Add queue-based media download system with rate limiting
- Add download_status, retry_after, queued_at columns to mls_media table - Add mls_media_log table for download attempt tracking - Rewrite media handler to queue downloads instead of immediate download - Add 700ms delay between downloads (25% buffer over 2/sec limit) - Add 3-hour backoff for rate-limited (429) responses - Add max 5 attempts before marking as permanently failed - Add wp mls media command: status, process, reset, logs - Deprecate wp mls sync media in favor of wp mls media process - Update documentation with queue system details and cron examples Media downloads are now separate from property sync: 1. wp mls sync full/incremental - syncs properties, queues media 2. wp mls media process - downloads queued media with rate limiting Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -55,6 +55,13 @@ class MLS_DB {
|
||||
return $this->get_table_name(MLS_TABLE_SYNC_LOG);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get media log table name
|
||||
*/
|
||||
public function media_log_table() {
|
||||
return $this->get_table_name(MLS_TABLE_MEDIA_LOG);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create all database tables
|
||||
*/
|
||||
@@ -163,6 +170,9 @@ class MLS_DB {
|
||||
downloaded_at DATETIME DEFAULT NULL,
|
||||
download_attempts INT(3) DEFAULT 0,
|
||||
download_error TEXT DEFAULT NULL,
|
||||
retry_after DATETIME DEFAULT NULL,
|
||||
queued_at DATETIME DEFAULT NULL,
|
||||
download_status VARCHAR(20) DEFAULT 'pending',
|
||||
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
@@ -170,7 +180,10 @@ class MLS_DB {
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY listing_media (listing_key, media_key),
|
||||
KEY listing_key (listing_key),
|
||||
KEY media_order (media_order)
|
||||
KEY media_order (media_order),
|
||||
KEY download_status (download_status),
|
||||
KEY retry_after (retry_after),
|
||||
KEY queued_at (queued_at)
|
||||
) {$charset_collate};";
|
||||
|
||||
dbDelta($sql_media);
|
||||
@@ -239,6 +252,29 @@ class MLS_DB {
|
||||
) {$charset_collate};";
|
||||
|
||||
dbDelta($sql_sync_log);
|
||||
|
||||
// Media download log table
|
||||
$table_media_log = $wpdb->prefix . MLS_TABLE_MEDIA_LOG;
|
||||
$sql_media_log = "CREATE TABLE {$table_media_log} (
|
||||
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
media_id BIGINT(20) UNSIGNED NOT NULL,
|
||||
listing_key VARCHAR(50) NOT NULL,
|
||||
media_key VARCHAR(100) NOT NULL,
|
||||
action VARCHAR(30) NOT NULL,
|
||||
status_code INT(5) DEFAULT NULL,
|
||||
response_time_ms INT(11) DEFAULT NULL,
|
||||
error_message TEXT DEFAULT NULL,
|
||||
url VARCHAR(1000) DEFAULT NULL,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
|
||||
PRIMARY KEY (id),
|
||||
KEY media_id (media_id),
|
||||
KEY listing_key (listing_key),
|
||||
KEY action (action),
|
||||
KEY created_at (created_at)
|
||||
) {$charset_collate};";
|
||||
|
||||
dbDelta($sql_media_log);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -253,6 +289,7 @@ class MLS_DB {
|
||||
MLS_TABLE_SYNC_STATE,
|
||||
MLS_TABLE_RATE_LIMITS,
|
||||
MLS_TABLE_SYNC_LOG,
|
||||
MLS_TABLE_MEDIA_LOG,
|
||||
);
|
||||
|
||||
foreach ($tables as $table) {
|
||||
|
||||
Reference in New Issue
Block a user