Hanson.xyz Dev
6eadf3d266
- Add download_status, retry_after, queued_at columns to mls_media table - Add mls_media_log table for download attempt tracking - Rewrite media handler to queue downloads instead of immediate download - Add 700ms delay between downloads (25% buffer over 2/sec limit) - Add 3-hour backoff for rate-limited (429) responses - Add max 5 attempts before marking as permanently failed - Add wp mls media command: status, process, reset, logs - Deprecate wp mls sync media in favor of wp mls media process - Update documentation with queue system details and cron examples Media downloads are now separate from property sync: 1. wp mls sync full/incremental - syncs properties, queues media 2. wp mls media process - downloads queued media with rate limiting Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
364 lines
12 KiB
PHP
364 lines
12 KiB
PHP
<?php
|
|
/**
|
|
* Database handler class
|
|
*/
|
|
|
|
if (!defined('ABSPATH')) {
|
|
exit;
|
|
}
|
|
|
|
class MLS_DB {
|
|
|
|
/**
|
|
* Get table name with prefix
|
|
*
|
|
* @param string $table Table name constant
|
|
* @return string Full table name
|
|
*/
|
|
public function get_table_name($table) {
|
|
global $wpdb;
|
|
return $wpdb->prefix . $table;
|
|
}
|
|
|
|
/**
|
|
* Get properties table name
|
|
*/
|
|
public function properties_table() {
|
|
return $this->get_table_name(MLS_TABLE_PROPERTIES);
|
|
}
|
|
|
|
/**
|
|
* Get media table name
|
|
*/
|
|
public function media_table() {
|
|
return $this->get_table_name(MLS_TABLE_MEDIA);
|
|
}
|
|
|
|
/**
|
|
* Get sync state table name
|
|
*/
|
|
public function sync_state_table() {
|
|
return $this->get_table_name(MLS_TABLE_SYNC_STATE);
|
|
}
|
|
|
|
/**
|
|
* Get rate limits table name
|
|
*/
|
|
public function rate_limits_table() {
|
|
return $this->get_table_name(MLS_TABLE_RATE_LIMITS);
|
|
}
|
|
|
|
/**
|
|
* Get sync log table name
|
|
*/
|
|
public function sync_log_table() {
|
|
return $this->get_table_name(MLS_TABLE_SYNC_LOG);
|
|
}
|
|
|
|
/**
|
|
* Get media log table name
|
|
*/
|
|
public function media_log_table() {
|
|
return $this->get_table_name(MLS_TABLE_MEDIA_LOG);
|
|
}
|
|
|
|
/**
|
|
* Create all database tables
|
|
*/
|
|
public static function create_tables() {
|
|
global $wpdb;
|
|
|
|
$charset_collate = $wpdb->get_charset_collate();
|
|
|
|
require_once ABSPATH . 'wp-admin/includes/upgrade.php';
|
|
|
|
// Properties table
|
|
$table_properties = $wpdb->prefix . MLS_TABLE_PROPERTIES;
|
|
$sql_properties = "CREATE TABLE {$table_properties} (
|
|
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
listing_key VARCHAR(50) NOT NULL,
|
|
listing_id VARCHAR(50) NOT NULL,
|
|
originating_system VARCHAR(50) DEFAULT 'northstar',
|
|
|
|
standard_status VARCHAR(30) NOT NULL,
|
|
mls_status VARCHAR(50) DEFAULT NULL,
|
|
mlg_can_view TINYINT(1) DEFAULT 1,
|
|
|
|
list_price DECIMAL(15,2) DEFAULT NULL,
|
|
original_list_price DECIMAL(15,2) DEFAULT NULL,
|
|
close_price DECIMAL(15,2) DEFAULT NULL,
|
|
|
|
street_number VARCHAR(20) DEFAULT NULL,
|
|
street_name VARCHAR(100) DEFAULT NULL,
|
|
street_suffix VARCHAR(30) DEFAULT NULL,
|
|
unit_number VARCHAR(20) DEFAULT NULL,
|
|
city VARCHAR(100) NOT NULL,
|
|
state_or_province VARCHAR(50) DEFAULT 'MN',
|
|
postal_code VARCHAR(20) DEFAULT NULL,
|
|
county VARCHAR(100) DEFAULT NULL,
|
|
latitude DECIMAL(10,8) DEFAULT NULL,
|
|
longitude DECIMAL(11,8) DEFAULT NULL,
|
|
|
|
property_type VARCHAR(50) DEFAULT NULL,
|
|
property_sub_type VARCHAR(50) DEFAULT NULL,
|
|
bedrooms_total INT(3) DEFAULT NULL,
|
|
bathrooms_total DECIMAL(4,1) DEFAULT NULL,
|
|
bathrooms_full INT(3) DEFAULT NULL,
|
|
bathrooms_half INT(3) DEFAULT NULL,
|
|
living_area INT(10) DEFAULT NULL,
|
|
lot_size_area DECIMAL(12,4) DEFAULT NULL,
|
|
lot_size_units VARCHAR(20) DEFAULT NULL,
|
|
year_built INT(4) DEFAULT NULL,
|
|
garage_spaces INT(3) DEFAULT NULL,
|
|
|
|
public_remarks TEXT DEFAULT NULL,
|
|
directions TEXT DEFAULT NULL,
|
|
|
|
list_agent_key VARCHAR(50) DEFAULT NULL,
|
|
list_agent_mls_id VARCHAR(50) DEFAULT NULL,
|
|
list_agent_name VARCHAR(150) DEFAULT NULL,
|
|
list_office_key VARCHAR(50) DEFAULT NULL,
|
|
list_office_mls_id VARCHAR(50) DEFAULT NULL,
|
|
list_office_name VARCHAR(150) DEFAULT NULL,
|
|
|
|
photos_count INT(5) DEFAULT 0,
|
|
modification_timestamp DATETIME NOT NULL,
|
|
photos_change_timestamp DATETIME DEFAULT NULL,
|
|
listing_contract_date DATE DEFAULT NULL,
|
|
close_date DATE DEFAULT NULL,
|
|
days_on_market INT(5) DEFAULT NULL,
|
|
|
|
raw_data LONGTEXT DEFAULT NULL,
|
|
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
|
|
PRIMARY KEY (id),
|
|
UNIQUE KEY listing_key (listing_key),
|
|
KEY listing_id (listing_id),
|
|
KEY standard_status (standard_status),
|
|
KEY city (city),
|
|
KEY property_type (property_type),
|
|
KEY modification_timestamp (modification_timestamp),
|
|
KEY list_price (list_price),
|
|
KEY mlg_can_view (mlg_can_view),
|
|
KEY bedrooms_total (bedrooms_total),
|
|
KEY county (county)
|
|
) {$charset_collate};";
|
|
|
|
dbDelta($sql_properties);
|
|
|
|
// Media table
|
|
$table_media = $wpdb->prefix . MLS_TABLE_MEDIA;
|
|
$sql_media = "CREATE TABLE {$table_media} (
|
|
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
listing_key VARCHAR(50) NOT NULL,
|
|
media_key VARCHAR(100) NOT NULL,
|
|
|
|
media_type VARCHAR(30) DEFAULT 'Photo',
|
|
media_order INT(5) DEFAULT 0,
|
|
media_url VARCHAR(1000) DEFAULT NULL,
|
|
|
|
local_path VARCHAR(500) DEFAULT NULL,
|
|
local_url VARCHAR(500) DEFAULT NULL,
|
|
file_size INT(11) DEFAULT NULL,
|
|
mime_type VARCHAR(50) DEFAULT NULL,
|
|
image_width INT(5) DEFAULT NULL,
|
|
image_height INT(5) DEFAULT NULL,
|
|
|
|
media_modification_timestamp DATETIME DEFAULT NULL,
|
|
downloaded_at DATETIME DEFAULT NULL,
|
|
download_attempts INT(3) DEFAULT 0,
|
|
download_error TEXT DEFAULT NULL,
|
|
retry_after DATETIME DEFAULT NULL,
|
|
queued_at DATETIME DEFAULT NULL,
|
|
download_status VARCHAR(20) DEFAULT 'pending',
|
|
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
|
|
PRIMARY KEY (id),
|
|
UNIQUE KEY listing_media (listing_key, media_key),
|
|
KEY listing_key (listing_key),
|
|
KEY media_order (media_order),
|
|
KEY download_status (download_status),
|
|
KEY retry_after (retry_after),
|
|
KEY queued_at (queued_at)
|
|
) {$charset_collate};";
|
|
|
|
dbDelta($sql_media);
|
|
|
|
// Sync state table
|
|
$table_sync_state = $wpdb->prefix . MLS_TABLE_SYNC_STATE;
|
|
$sql_sync_state = "CREATE TABLE {$table_sync_state} (
|
|
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
sync_type VARCHAR(30) NOT NULL,
|
|
entity_type VARCHAR(30) NOT NULL DEFAULT 'Property',
|
|
|
|
status VARCHAR(20) DEFAULT 'pending',
|
|
started_at DATETIME DEFAULT NULL,
|
|
completed_at DATETIME DEFAULT NULL,
|
|
|
|
last_modification_timestamp DATETIME DEFAULT NULL,
|
|
last_next_link VARCHAR(2000) DEFAULT NULL,
|
|
records_processed INT(11) DEFAULT 0,
|
|
records_created INT(11) DEFAULT 0,
|
|
records_updated INT(11) DEFAULT 0,
|
|
records_deleted INT(11) DEFAULT 0,
|
|
|
|
error_count INT(11) DEFAULT 0,
|
|
last_error TEXT DEFAULT NULL,
|
|
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
|
|
PRIMARY KEY (id),
|
|
KEY sync_type_entity (sync_type, entity_type),
|
|
KEY status (status)
|
|
) {$charset_collate};";
|
|
|
|
dbDelta($sql_sync_state);
|
|
|
|
// Rate limits table
|
|
$table_rate_limits = $wpdb->prefix . MLS_TABLE_RATE_LIMITS;
|
|
$sql_rate_limits = "CREATE TABLE {$table_rate_limits} (
|
|
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
window_type VARCHAR(20) NOT NULL,
|
|
window_start DATETIME NOT NULL,
|
|
request_count INT(11) DEFAULT 0,
|
|
bytes_transferred BIGINT(20) DEFAULT 0,
|
|
|
|
PRIMARY KEY (id),
|
|
UNIQUE KEY window_type_start (window_type, window_start),
|
|
KEY window_start (window_start)
|
|
) {$charset_collate};";
|
|
|
|
dbDelta($sql_rate_limits);
|
|
|
|
// Sync log table
|
|
$table_sync_log = $wpdb->prefix . MLS_TABLE_SYNC_LOG;
|
|
$sql_sync_log = "CREATE TABLE {$table_sync_log} (
|
|
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
sync_state_id BIGINT(20) UNSIGNED DEFAULT NULL,
|
|
level VARCHAR(20) DEFAULT 'info',
|
|
message TEXT NOT NULL,
|
|
context LONGTEXT DEFAULT NULL,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
|
|
PRIMARY KEY (id),
|
|
KEY sync_state_id (sync_state_id),
|
|
KEY level (level),
|
|
KEY created_at (created_at)
|
|
) {$charset_collate};";
|
|
|
|
dbDelta($sql_sync_log);
|
|
|
|
// Media download log table
|
|
$table_media_log = $wpdb->prefix . MLS_TABLE_MEDIA_LOG;
|
|
$sql_media_log = "CREATE TABLE {$table_media_log} (
|
|
id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT,
|
|
media_id BIGINT(20) UNSIGNED NOT NULL,
|
|
listing_key VARCHAR(50) NOT NULL,
|
|
media_key VARCHAR(100) NOT NULL,
|
|
action VARCHAR(30) NOT NULL,
|
|
status_code INT(5) DEFAULT NULL,
|
|
response_time_ms INT(11) DEFAULT NULL,
|
|
error_message TEXT DEFAULT NULL,
|
|
url VARCHAR(1000) DEFAULT NULL,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
|
|
PRIMARY KEY (id),
|
|
KEY media_id (media_id),
|
|
KEY listing_key (listing_key),
|
|
KEY action (action),
|
|
KEY created_at (created_at)
|
|
) {$charset_collate};";
|
|
|
|
dbDelta($sql_media_log);
|
|
}
|
|
|
|
/**
|
|
* Drop all tables (for uninstall)
|
|
*/
|
|
public static function drop_tables() {
|
|
global $wpdb;
|
|
|
|
$tables = array(
|
|
MLS_TABLE_PROPERTIES,
|
|
MLS_TABLE_MEDIA,
|
|
MLS_TABLE_SYNC_STATE,
|
|
MLS_TABLE_RATE_LIMITS,
|
|
MLS_TABLE_SYNC_LOG,
|
|
MLS_TABLE_MEDIA_LOG,
|
|
);
|
|
|
|
foreach ($tables as $table) {
|
|
$table_name = $wpdb->prefix . $table;
|
|
$wpdb->query("DROP TABLE IF EXISTS {$table_name}");
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Truncate all data tables (keep structure)
|
|
*/
|
|
public function truncate_data() {
|
|
global $wpdb;
|
|
|
|
$wpdb->query("TRUNCATE TABLE {$this->properties_table()}");
|
|
$wpdb->query("TRUNCATE TABLE {$this->media_table()}");
|
|
$wpdb->query("TRUNCATE TABLE {$this->sync_state_table()}");
|
|
$wpdb->query("TRUNCATE TABLE {$this->sync_log_table()}");
|
|
}
|
|
|
|
/**
|
|
* Get database statistics
|
|
*/
|
|
public function get_stats() {
|
|
global $wpdb;
|
|
|
|
$stats = array(
|
|
'total_properties' => 0,
|
|
'active_properties' => 0,
|
|
'pending_properties' => 0,
|
|
'sold_properties' => 0,
|
|
'total_media' => 0,
|
|
'downloaded_media' => 0,
|
|
);
|
|
|
|
// Property counts
|
|
$stats['total_properties'] = (int) $wpdb->get_var(
|
|
"SELECT COUNT(*) FROM {$this->properties_table()} WHERE mlg_can_view = 1"
|
|
);
|
|
|
|
$status_counts = $wpdb->get_results(
|
|
"SELECT standard_status, COUNT(*) as count
|
|
FROM {$this->properties_table()}
|
|
WHERE mlg_can_view = 1
|
|
GROUP BY standard_status",
|
|
OBJECT_K
|
|
);
|
|
|
|
if (isset($status_counts['Active'])) {
|
|
$stats['active_properties'] = (int) $status_counts['Active']->count;
|
|
}
|
|
if (isset($status_counts['Pending'])) {
|
|
$stats['pending_properties'] = (int) $status_counts['Pending']->count;
|
|
}
|
|
if (isset($status_counts['Closed']) || isset($status_counts['Sold'])) {
|
|
$stats['sold_properties'] = (int) ($status_counts['Closed']->count ?? 0)
|
|
+ (int) ($status_counts['Sold']->count ?? 0);
|
|
}
|
|
|
|
// Media counts
|
|
$stats['total_media'] = (int) $wpdb->get_var(
|
|
"SELECT COUNT(*) FROM {$this->media_table()}"
|
|
);
|
|
|
|
$stats['downloaded_media'] = (int) $wpdb->get_var(
|
|
"SELECT COUNT(*) FROM {$this->media_table()} WHERE local_path IS NOT NULL"
|
|
);
|
|
|
|
return $stats;
|
|
}
|
|
}
|