Added TMCH cron and database patches

getnamingo · Dec 15, 2023 · b4ccc2b · b4ccc2b
1 parent f9ae818
commit b4ccc2b
Show file tree

Hide file tree

Showing 5 changed files with 192 additions and 30 deletions.
diff --git a/automation/config.php.dist b/automation/config.php.dist
@@ -63,4 +63,11 @@ return [
     'mailer_sms' => 'twilio', // telesign, plivo, vonage, clickatell are also available
     'mailer_sms_account' => 'YOUR_ACCOUNT_SID/USERNAME',
     'mailer_sms_auth' => 'YOUR_AUTH_TOKEN/PASSWORD',
+
+    // TMCH Configuration
+    'tmch_path' => '/tmp/',
+    'tmch_smdrl_user' => 'your_username',
+    'tmch_smdrl_pass' => 'your_password',
+    'tmch_dnl_user' => 'your_username',
+    'tmch_dnl_pass' => 'your_password',
 ];
diff --git a/automation/crontab.example b/automation/crontab.example
@@ -27,8 +27,8 @@
 # run auto-clean-unused-contact-and-host.php every day
 5 0 * * * root /usr/bin/php8.2 /opt/registry/automation/auto-clean-unused-contact-and-host.php
 
-# run tmch.php every day
-5 0 * * * root /usr/bin/php8.2 /opt/registry/automation/tmch.php
+# run tmch.php twice a day
+0 0,12 * * * root /usr/bin/php8.2 /opt/registry/automation/tmch.php
 
 # run escrow.php every day
 5 0 * * * root /usr/bin/php8.2 /opt/registry/automation/escrow.php

diff --git a/automation/tmch.php b/automation/tmch.php
@@ -1,48 +1,159 @@
 <?php
 
-$savePath = "/path/to/save/files/";  // Adjust this path as needed.
+$c = require_once 'config.php';
+require_once 'helpers.php';
+
+// Connect to the database
+$dsn = "{$c['db_type']}:host={$c['db_host']};dbname={$c['db_database']};port={$c['db_port']}";
+$logFilePath = '/var/log/namingo/tmch.log';
+$log = setupLogger($logFilePath, 'TMCH');
+$log->info('job started.');
+
+try {
+    $dbh = new PDO($dsn, $c['db_username'], $c['db_password']);
+    $dbh->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
+} catch (PDOException $e) {
+    $log->error('DB Connection failed: ' . $e->getMessage());
+}
+
+$savePath = $c['tmch_path'];
 
 $files = [
-    'smdrl' => 'https://ry.marksdb.org/smdrl/smdrl-latest.csv',
-    'surl' => 'https://test.ry.marksdb.org/dnl/surl-latest.csv',
+    'smdrl' => 'https://test.ry.marksdb.org/smdrl/smdrl-latest.csv',
     'dnl' => 'https://test.ry.marksdb.org/dnl/dnl-latest.csv',
     'tmch' => 'http://crl.icann.org/tmch.crl'
 ];
 
 // Configure the username and password for each URL.
 $credentials = [
-    'smdrl' => ['user' => 'username1', 'pass' => 'password1'],
-    'surl' => ['user' => 'username2', 'pass' => 'password2'],
-    'dnl' => ['user' => 'username3', 'pass' => 'password3']
-    // 'tmch' is not listed here since it doesn't require authentication.
+    'smdrl' => ['user' => $c['tmch_smdrl_user'], 'pass' => $c['tmch_smdrl_pass']],
+    'dnl' => ['user' => $c['tmch_dnl_user'], 'pass' => $c['tmch_dnl_pass']]
 ];
 
-foreach ($files as $key => $url) {
-    $ch = curl_init($url);
+try {
+    foreach ($files as $key => $url) {
+        $ch = curl_init($url);
 
-    // Check if credentials exist for this URL and set them.
-    if (isset($credentials[$key])) {
-        curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
-        curl_setopt($ch, CURLOPT_USERPWD, $credentials[$key]['user'] . ":" . $credentials[$key]['pass']);
-    }
+        // Check if credentials exist for this URL and set them.
+        if (isset($credentials[$key])) {
+            curl_setopt($ch, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
+            curl_setopt($ch, CURLOPT_USERPWD, $credentials[$key]['user'] . ":" . $credentials[$key]['pass']);
+        }
 
-    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-    curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
-    curl_setopt($ch, CURLOPT_MAXREDIRS, 10);
-    curl_setopt($ch, CURLOPT_TIMEOUT, 30);
+        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
+        curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
+        curl_setopt($ch, CURLOPT_MAXREDIRS, 10);
+        curl_setopt($ch, CURLOPT_TIMEOUT, 30);
 
-    $data = curl_exec($ch);
+        $data = curl_exec($ch);
 
-    if ($data === false) {
-        echo "Failed to download $url. Error: " . curl_error($ch) . "\n";
-    } else {
-        $filePath = $savePath . basename($url);
-        if (file_put_contents($filePath, $data)) {
-            echo "Successfully downloaded $url to $filePath\n";
+        if ($data === false) {
+            $log->error("Failed to download $url. Error: " . curl_error($ch));
         } else {
-            echo "Failed to save the downloaded file to $filePath\n";
+            $timestamp = date('YmdHis');
+            $filePath = $savePath . $timestamp . '_' . basename($url);
+            if (file_put_contents($filePath, $data)) {
+                $log->info("Successfully downloaded $url to $filePath");
+            } else {
+                $log->info("Failed to save the downloaded file to $filePath");
+            }
+        }
+
+        curl_close($ch);
+
+        // Clear existing table data
+        clearTableData($dbh, $key);
+
+        // Depending on the file type, call different function to handle and insert data
+        if ($key === 'smdrl') {
+            insertSmdrlData($dbh, $filePath, $log);
+        } elseif ($key === 'dnl') {
+            insertDnlData($dbh, $filePath);
+        } elseif ($key === 'tmch') {
+            insertTmchData($dbh, $filePath);
+        }
+    }
+    $log->info('job finished successfully.');
+} catch (PDOException $e) {
+    $log->error('Database error: ' . $e->getMessage());
+} catch (Throwable $e) {
+    $log->error('Error: ' . $e->getMessage());
+}
+
+function clearTableData($dbh, $key) {
+    $tableMap = [
+        'smdrl' => 'tmch_revocation',
+        'dnl' => 'tmch_claims',
+        'tmch' => 'tmch_crl'
+    ];
+
+    if (isset($tableMap[$key])) {
+        $dbh->exec("TRUNCATE TABLE " . $tableMap[$key]);
+    }
+}
+
+function insertSmdrlData($dbh, $filePath, $log) {
+    if (($handle = fopen($filePath, "r")) !== FALSE) {
+        // Skip the first two header rows
+        for ($i = 0; $i < 2; $i++) {
+            fgetcsv($handle);
         }
+
+        while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
+            if (count($data) >= 2) {
+                // Convert ISO 8601 format to MySQL DATETIME(3) format
+                try {
+                    $date = new DateTime($data[1]);
+                    $formattedDate = $date->format('Y-m-d H:i:s.v'); // Format for MySQL DATETIME(3)
+                } catch (Exception $e) {
+                    $log->info('Date conversion error:' . $e->getMessage());
+                    continue; // Skip this record or handle it as needed
+                }
+
+                // Assume data format: smd_id,insertion_datetime
+                $stmt = $dbh->prepare("INSERT INTO tmch_revocation (smd_id, revocation_time) VALUES (?, ?)");
+                $stmt->execute([$data[0], $formattedDate]);
+            }
+        }
+        fclose($handle);
     }
+}
 
-    curl_close($ch);
+function insertDnlData($dbh, $filePath) {
+    if (($handle = fopen($filePath, "r")) !== FALSE) {
+        // Skip the first two header rows
+        for ($i = 0; $i < 2; $i++) {
+            fgetcsv($handle);
+        }
+
+        while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
+            if (count($data) >= 3) {
+                // Convert ISO 8601 format to MySQL DATETIME(3) format
+                try {
+                    $date = new DateTime($data[2]);
+                    $formattedDate = $date->format('Y-m-d H:i:s.v'); // Format for MySQL DATETIME(3)
+                } catch (Exception $e) {
+                    $log->info('Date conversion error:' . $e->getMessage());
+                    continue; // Skip this record or handle it as needed
+                }
+
+                // Assuming data format
+                $stmt = $dbh->prepare("INSERT INTO tmch_claims (domain_label, claim_key, insert_time) VALUES (?, ?, ?)");
+                $stmt->execute([$data[0], $data[1], $formattedDate]);
+            }
+        }
+        fclose($handle);
+    }
+}
+
+function insertTmchData($dbh, $filePath) {
+    $content = file_get_contents($filePath);
+    if ($content !== false) {
+        // Get the current date-time in the format accepted by both MySQL and PostgreSQL
+        $currentDateTime = (new DateTime())->format('Y-m-d H:i:s');
+
+        // Insert the whole content into the database
+        $stmt = $dbh->prepare("INSERT INTO tmch_crl (content, url, update_timestamp) VALUES (?, ?, ?)");
+        $stmt->execute([$content, $filePath, $currentDateTime]);
+    }
 }
diff --git a/database/registry.mariadb.sql b/database/registry.mariadb.sql
@@ -672,6 +672,28 @@ CREATE TABLE IF NOT EXISTS `registry`.`ticket_responses` (
     FOREIGN KEY (ticket_id) REFERENCES support_tickets(id)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='Ticket Responses';
 
+CREATE TABLE IF NOT EXISTS `registry`.`tmch_claims` (
+    `id` INT(11) UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+    `domain_label` VARCHAR(100) NOT NULL,
+    `claim_key` VARCHAR(200) NOT NULL,
+    `insert_time` datetime(3) NOT NULL,
+    UNIQUE KEY `tmch_claims_1` (`domain_label`,`claim_key`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='TMCH Claims';
+
+CREATE TABLE IF NOT EXISTS `registry`.`tmch_revocation` (
+    `id` INT(11) UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+    `smd_id` VARCHAR(100) NOT NULL,
+    `revocation_time` datetime(3) NOT NULL,
+    UNIQUE KEY `tmch_revocation_1` (`smd_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='TMCH Revocation';
+
+CREATE TABLE IF NOT EXISTS `registry`.`tmch_crl` (
+    `id` INT(11) UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+    `content` TEXT NOT NULL,
+    `url` VARCHAR(255) NOT NULL,
+    `update_timestamp` datetime(3) NOT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci COMMENT='TMCH Crl';
+
 INSERT INTO `registry`.`domain_tld` VALUES('1','.TEST','/^(?!-)(?!.*--)[A-Z0-9-]{1,63}(?<!-)(\.(?!-)(?!.*--)[A-Z0-9-]{1,63}(?<!-))*$/i','0');
 INSERT INTO `registry`.`domain_tld` VALUES('2','.COM.TEST','/^(?!-)(?!.*--)[A-Z0-9-]{1,63}(?<!-)(\.(?!-)(?!.*--)[A-Z0-9-]{1,63}(?<!-))*$/i','0');
 

diff --git a/database/registry.postgres.sql b/database/registry.postgres.sql
@@ -642,7 +642,7 @@ CREATE TABLE registry.support_tickets (
     FOREIGN KEY (category_id) REFERENCES registry.ticket_categories(id)
 );
 
-CREATE TABLE ticket_responses (
+CREATE TABLE registry.ticket_responses (
     "id" SERIAL PRIMARY KEY,
     "ticket_id" INTEGER NOT NULL,
     "responder_id" INTEGER NOT NULL,
@@ -651,6 +651,28 @@ CREATE TABLE ticket_responses (
     FOREIGN KEY (ticket_id) REFERENCES support_tickets(id)
 );
 
+CREATE TABLE registry.tmch_claims (
+    "id" SERIAL PRIMARY KEY,
+    "domain_label" VARCHAR(100) NOT NULL,
+    "claim_key" VARCHAR(200) NOT NULL,
+    "insert_time" TIMESTAMP(3) NOT NULL,
+    CONSTRAINT tmch_claims_unique UNIQUE (claim_key, domain_label)
+);
+
+CREATE TABLE registry.tmch_revocation (
+    "id" SERIAL PRIMARY KEY,
+    "smd_id" VARCHAR(100) NOT NULL,
+    "revocation_time" TIMESTAMP(3) NOT NULL,
+    CONSTRAINT tmch_revocation_unique UNIQUE (smd_id)
+);
+
+CREATE TABLE registry.tmch_crl (
+    "id" SERIAL PRIMARY KEY,
+    "content" TEXT NOT NULL,
+    "url" VARCHAR(255) NOT NULL,
+    "update_timestamp" TIMESTAMP(3) NOT NULL
+);
+
 INSERT INTO registry.domain_tld VALUES('1','.TEST','/^(?!-)(?!.*--)[A-Z0-9-]{1,63}(?<!-)(\.(?!-)(?!.*--)[A-Z0-9-]{1,63}(?<!-))*$/i','0');
 INSERT INTO registry.domain_tld VALUES('2','.COM.TEST','/^(?!-)(?!.*--)[A-Z0-9-]{1,63}(?<!-)(\.(?!-)(?!.*--)[A-Z0-9-]{1,63}(?<!-))*$/i','0');