Add MLS by HansonXyz plugin for MLS Grid API integration

Features:
- Full sync of NorthStar MLS properties via MLS Grid API v2
- Incremental sync using ModificationTimestamp
- Local media download and storage
- Rate limit compliance (2 req/sec, 7200/hr, 40000/day)
- Sync state tracking with resume capability
- WP-CLI commands: test, sync, status, stats, cache
- Admin settings page with manual sync triggers
- Public API functions: mls_get_properties, mls_get_property, etc.

Database tables:
- mls_properties: Listing data with full field mapping
- mls_media: Downloaded images
- mls_sync_state: Sync progress tracking
- mls_rate_limits: API usage tracking
- mls_sync_log: Debug logging

Documentation:
- docs/CLAUDE.md: AI development guide
- docs/API.md: MLS Grid API reference
- docs/USAGE.md: User documentation

Tested: Connection, auth, sync 10 records, media download verified

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Hanson.xyz Dev
2025-12-14 21:24:38 -06:00
parent ec5a309555
commit 6556479417
18 changed files with 5324 additions and 10 deletions
@@ -0,0 +1,694 @@
<?php
/**
* MLS Sync Engine
*
* Handles synchronization of MLS data including:
* - Full initial import
* - Incremental updates
* - Delete handling (MlgCanView = false)
* - Sync state tracking for resume capability
*/
if (!defined('ABSPATH')) {
exit;
}
class MLS_Sync_Engine {
/**
* Sync types
*/
const TYPE_FULL = 'full';
const TYPE_INCREMENTAL = 'incremental';
const TYPE_MEDIA = 'media';
/**
* Sync statuses
*/
const STATUS_PENDING = 'pending';
const STATUS_RUNNING = 'running';
const STATUS_COMPLETED = 'completed';
const STATUS_FAILED = 'failed';
const STATUS_PAUSED = 'paused';
/**
* Database instance
*/
private $db;
/**
* API client instance
*/
private $api_client;
/**
* Media handler instance
*/
private $media_handler;
/**
* Logger instance
*/
private $logger;
/**
* Current sync state ID
*/
private $sync_state_id = null;
/**
* Stats for current sync
*/
private $stats = array(
'processed' => 0,
'created' => 0,
'updated' => 0,
'deleted' => 0,
'errors' => 0,
);
/**
* Constructor
*/
public function __construct(MLS_DB $db, MLS_API_Client $api_client, MLS_Media_Handler $media_handler, MLS_Logger $logger) {
$this->db = $db;
$this->api_client = $api_client;
$this->media_handler = $media_handler;
$this->logger = $logger;
}
/**
* Run full sync
*
* @param bool $dry_run If true, don't make changes
* @param int|null $limit Max records to process
* @param callable|null $progress_callback Callback for progress updates
* @return array Sync results
*/
public function run_full_sync($dry_run = false, $limit = null, $progress_callback = null) {
$this->logger->info('Starting full sync', array('dry_run' => $dry_run, 'limit' => $limit));
// Create sync state record
if (!$dry_run) {
$this->sync_state_id = $this->create_sync_state(self::TYPE_FULL);
$this->logger->set_sync_state($this->sync_state_id);
}
$this->stats = array(
'processed' => 0,
'created' => 0,
'updated' => 0,
'deleted' => 0,
'errors' => 0,
);
try {
// Get first page of properties with media
$response = $this->api_client->get_properties_for_sync(null, 'Media', $limit ? min($limit, 1000) : null);
if (is_wp_error($response)) {
throw new Exception($response->get_error_message());
}
// Process pages
$continue = true;
while ($continue && isset($response['value'])) {
foreach ($response['value'] as $property) {
if ($limit && $this->stats['processed'] >= $limit) {
$continue = false;
break;
}
$this->process_property($property, $dry_run);
if ($progress_callback) {
call_user_func($progress_callback, $this->stats);
}
}
// Check for next page
if ($continue && isset($response['@odata.nextLink'])) {
// Save progress
if (!$dry_run) {
$this->update_sync_state(array(
'last_next_link' => $response['@odata.nextLink'],
'records_processed' => $this->stats['processed'],
'records_created' => $this->stats['created'],
'records_updated' => $this->stats['updated'],
));
}
$response = $this->api_client->get_next_page($response['@odata.nextLink']);
if (is_wp_error($response)) {
throw new Exception($response->get_error_message());
}
} else {
$continue = false;
}
}
// Mark sync as completed
if (!$dry_run) {
$this->update_sync_state(array(
'status' => self::STATUS_COMPLETED,
'completed_at' => current_time('mysql'),
'records_processed' => $this->stats['processed'],
'records_created' => $this->stats['created'],
'records_updated' => $this->stats['updated'],
'records_deleted' => $this->stats['deleted'],
));
// Update last sync time
$options = mls_plugin()->get_options();
$options->update_last_sync('full');
}
$this->logger->info('Full sync completed', $this->stats);
} catch (Exception $e) {
$this->logger->error('Full sync failed', array('error' => $e->getMessage()));
if (!$dry_run) {
$this->update_sync_state(array(
'status' => self::STATUS_FAILED,
'last_error' => $e->getMessage(),
'error_count' => $this->stats['errors'] + 1,
));
}
return array(
'success' => false,
'error' => $e->getMessage(),
'stats' => $this->stats,
);
}
return array(
'success' => true,
'stats' => $this->stats,
);
}
/**
* Run incremental sync
*
* @param bool $dry_run If true, don't make changes
* @param callable|null $progress_callback Callback for progress updates
* @return array Sync results
*/
public function run_incremental_sync($dry_run = false, $progress_callback = null) {
// Get last modification timestamp
$last_timestamp = $this->get_last_modification_timestamp();
if (!$last_timestamp) {
$this->logger->info('No previous sync found, running full sync instead');
return $this->run_full_sync($dry_run, null, $progress_callback);
}
$this->logger->info('Starting incremental sync', array(
'since' => $last_timestamp,
'dry_run' => $dry_run,
));
if (!$dry_run) {
$this->sync_state_id = $this->create_sync_state(self::TYPE_INCREMENTAL);
$this->logger->set_sync_state($this->sync_state_id);
}
$this->stats = array(
'processed' => 0,
'created' => 0,
'updated' => 0,
'deleted' => 0,
'errors' => 0,
);
try {
// Get modified properties (including those marked for deletion)
$response = $this->api_client->get_properties_since($last_timestamp, 'Media');
if (is_wp_error($response)) {
throw new Exception($response->get_error_message());
}
// Process pages
while (isset($response['value'])) {
foreach ($response['value'] as $property) {
$this->process_property($property, $dry_run);
if ($progress_callback) {
call_user_func($progress_callback, $this->stats);
}
}
// Check for next page
if (isset($response['@odata.nextLink'])) {
$response = $this->api_client->get_next_page($response['@odata.nextLink']);
if (is_wp_error($response)) {
throw new Exception($response->get_error_message());
}
} else {
break;
}
}
// Mark sync as completed
if (!$dry_run) {
$this->update_sync_state(array(
'status' => self::STATUS_COMPLETED,
'completed_at' => current_time('mysql'),
'records_processed' => $this->stats['processed'],
'records_created' => $this->stats['created'],
'records_updated' => $this->stats['updated'],
'records_deleted' => $this->stats['deleted'],
));
$options = mls_plugin()->get_options();
$options->update_last_sync('incremental');
}
$this->logger->info('Incremental sync completed', $this->stats);
} catch (Exception $e) {
$this->logger->error('Incremental sync failed', array('error' => $e->getMessage()));
if (!$dry_run) {
$this->update_sync_state(array(
'status' => self::STATUS_FAILED,
'last_error' => $e->getMessage(),
));
}
return array(
'success' => false,
'error' => $e->getMessage(),
'stats' => $this->stats,
);
}
return array(
'success' => true,
'stats' => $this->stats,
);
}
/**
* Resume an interrupted sync
*
* @param int $sync_state_id Sync state ID to resume
* @param callable|null $progress_callback Progress callback
* @return array Sync results
*/
public function resume_sync($sync_state_id, $progress_callback = null) {
global $wpdb;
$state = $wpdb->get_row($wpdb->prepare(
"SELECT * FROM {$this->db->sync_state_table()} WHERE id = %d",
$sync_state_id
));
if (!$state) {
return array(
'success' => false,
'error' => 'Sync state not found',
);
}
if ($state->status === self::STATUS_COMPLETED) {
return array(
'success' => false,
'error' => 'Sync already completed',
);
}
$this->sync_state_id = $sync_state_id;
$this->logger->set_sync_state($sync_state_id);
$this->logger->info('Resuming sync', array('sync_state_id' => $sync_state_id));
// Load existing stats
$this->stats = array(
'processed' => (int) $state->records_processed,
'created' => (int) $state->records_created,
'updated' => (int) $state->records_updated,
'deleted' => (int) $state->records_deleted,
'errors' => (int) $state->error_count,
);
// Update status to running
$this->update_sync_state(array('status' => self::STATUS_RUNNING));
try {
// Resume from last next_link
if ($state->last_next_link) {
$response = $this->api_client->get_next_page($state->last_next_link);
} else {
// Start fresh
$response = $this->api_client->get_properties_for_sync(
$state->last_modification_timestamp,
'Media'
);
}
if (is_wp_error($response)) {
throw new Exception($response->get_error_message());
}
// Process remaining pages
while (isset($response['value'])) {
foreach ($response['value'] as $property) {
$this->process_property($property, false);
if ($progress_callback) {
call_user_func($progress_callback, $this->stats);
}
}
if (isset($response['@odata.nextLink'])) {
$this->update_sync_state(array(
'last_next_link' => $response['@odata.nextLink'],
'records_processed' => $this->stats['processed'],
));
$response = $this->api_client->get_next_page($response['@odata.nextLink']);
if (is_wp_error($response)) {
throw new Exception($response->get_error_message());
}
} else {
break;
}
}
$this->update_sync_state(array(
'status' => self::STATUS_COMPLETED,
'completed_at' => current_time('mysql'),
'records_processed' => $this->stats['processed'],
'records_created' => $this->stats['created'],
'records_updated' => $this->stats['updated'],
'records_deleted' => $this->stats['deleted'],
));
$this->logger->info('Resume sync completed', $this->stats);
} catch (Exception $e) {
$this->logger->error('Resume sync failed', array('error' => $e->getMessage()));
$this->update_sync_state(array(
'status' => self::STATUS_FAILED,
'last_error' => $e->getMessage(),
));
return array(
'success' => false,
'error' => $e->getMessage(),
'stats' => $this->stats,
);
}
return array(
'success' => true,
'stats' => $this->stats,
);
}
/**
* Process a single property record
*
* @param array $property Property data from API
* @param bool $dry_run If true, don't make changes
*/
private function process_property($property, $dry_run = false) {
global $wpdb;
$this->stats['processed']++;
$listing_key = $property['ListingKey'] ?? null;
if (!$listing_key) {
$this->stats['errors']++;
$this->logger->warning('Property missing ListingKey', array('property' => $property));
return;
}
// Check MlgCanView - if false, delete the record
$can_view = $property['MlgCanView'] ?? true;
if (!$can_view) {
if (!$dry_run) {
$this->delete_property($listing_key);
}
$this->stats['deleted']++;
return;
}
// Check if property exists
$existing = $wpdb->get_var($wpdb->prepare(
"SELECT id FROM {$this->db->properties_table()} WHERE listing_key = %s",
$listing_key
));
// Prepare data for insert/update
$data = $this->map_property_data($property);
if ($dry_run) {
if ($existing) {
$this->stats['updated']++;
} else {
$this->stats['created']++;
}
return;
}
if ($existing) {
// Update existing
$wpdb->update(
$this->db->properties_table(),
$data,
array('listing_key' => $listing_key)
);
$this->stats['updated']++;
} else {
// Insert new
$data['listing_key'] = $listing_key;
$data['created_at'] = current_time('mysql');
$wpdb->insert($this->db->properties_table(), $data);
$this->stats['created']++;
}
// Process media if present
if (isset($property['Media']) && is_array($property['Media'])) {
$this->media_handler->sync_property_media($listing_key, $property['Media']);
}
}
/**
* Map API property data to database columns
*
* @param array $property API property data
* @return array Mapped data for database
*/
private function map_property_data($property) {
return array(
'listing_id' => $property['ListingId'] ?? null,
'originating_system' => $property['OriginatingSystemName'] ?? 'northstar',
'standard_status' => $property['StandardStatus'] ?? null,
'mls_status' => $property['MlsStatus'] ?? null,
'mlg_can_view' => isset($property['MlgCanView']) ? ($property['MlgCanView'] ? 1 : 0) : 1,
'list_price' => $property['ListPrice'] ?? null,
'original_list_price' => $property['OriginalListPrice'] ?? null,
'close_price' => $property['ClosePrice'] ?? null,
'street_number' => $property['StreetNumber'] ?? null,
'street_name' => $property['StreetName'] ?? null,
'street_suffix' => $property['StreetSuffix'] ?? null,
'unit_number' => $property['UnitNumber'] ?? null,
'city' => $property['City'] ?? null,
'state_or_province' => $property['StateOrProvince'] ?? 'MN',
'postal_code' => $property['PostalCode'] ?? null,
'county' => $property['CountyOrParish'] ?? null,
'latitude' => $property['Latitude'] ?? null,
'longitude' => $property['Longitude'] ?? null,
'property_type' => $property['PropertyType'] ?? null,
'property_sub_type' => $property['PropertySubType'] ?? null,
'bedrooms_total' => $property['BedroomsTotal'] ?? null,
'bathrooms_total' => $property['BathroomsTotalInteger'] ?? null,
'bathrooms_full' => $property['BathroomsFull'] ?? null,
'bathrooms_half' => $property['BathroomsHalf'] ?? null,
'living_area' => $property['LivingArea'] ?? null,
'lot_size_area' => $property['LotSizeArea'] ?? null,
'lot_size_units' => $property['LotSizeUnits'] ?? null,
'year_built' => $property['YearBuilt'] ?? null,
'garage_spaces' => $property['GarageSpaces'] ?? null,
'public_remarks' => $property['PublicRemarks'] ?? null,
'directions' => $property['Directions'] ?? null,
'list_agent_key' => $property['ListAgentKey'] ?? null,
'list_agent_mls_id' => $property['ListAgentMlsId'] ?? null,
'list_office_key' => $property['ListOfficeKey'] ?? null,
'list_office_mls_id' => $property['ListOfficeMlsId'] ?? null,
'list_office_name' => $property['ListOfficeName'] ?? null,
'photos_count' => $property['PhotosCount'] ?? 0,
'modification_timestamp' => $this->format_timestamp($property['ModificationTimestamp'] ?? null),
'photos_change_timestamp' => $this->format_timestamp($property['PhotosChangeTimestamp'] ?? null),
'listing_contract_date' => $this->format_date($property['ListingContractDate'] ?? null),
'close_date' => $this->format_date($property['CloseDate'] ?? null),
'days_on_market' => $property['DaysOnMarket'] ?? null,
'raw_data' => wp_json_encode($property),
'updated_at' => current_time('mysql'),
);
}
/**
* Format ISO 8601 timestamp to MySQL datetime
*
* @param string|null $timestamp ISO 8601 timestamp
* @return string|null MySQL datetime
*/
private function format_timestamp($timestamp) {
if (!$timestamp) {
return null;
}
$dt = new DateTime($timestamp);
return $dt->format('Y-m-d H:i:s');
}
/**
* Format date string to MySQL date
*
* @param string|null $date Date string
* @return string|null MySQL date
*/
private function format_date($date) {
if (!$date) {
return null;
}
return date('Y-m-d', strtotime($date));
}
/**
* Delete a property and its media
*
* @param string $listing_key Listing key
*/
private function delete_property($listing_key) {
global $wpdb;
// Delete media files
$this->media_handler->delete_property_media($listing_key);
// Delete from database
$wpdb->delete(
$this->db->properties_table(),
array('listing_key' => $listing_key)
);
$this->logger->debug('Deleted property', array('listing_key' => $listing_key));
}
/**
* Create a sync state record
*
* @param string $type Sync type
* @return int Sync state ID
*/
private function create_sync_state($type) {
global $wpdb;
$wpdb->insert(
$this->db->sync_state_table(),
array(
'sync_type' => $type,
'entity_type' => 'Property',
'status' => self::STATUS_RUNNING,
'started_at' => current_time('mysql'),
'created_at' => current_time('mysql'),
)
);
return $wpdb->insert_id;
}
/**
* Update sync state record
*
* @param array $data Data to update
*/
private function update_sync_state($data) {
global $wpdb;
if (!$this->sync_state_id) {
return;
}
$data['updated_at'] = current_time('mysql');
$wpdb->update(
$this->db->sync_state_table(),
$data,
array('id' => $this->sync_state_id)
);
}
/**
* Get last modification timestamp from synced data
*
* @return string|null ISO 8601 timestamp
*/
private function get_last_modification_timestamp() {
global $wpdb;
$timestamp = $wpdb->get_var(
"SELECT MAX(modification_timestamp) FROM {$this->db->properties_table()}"
);
if ($timestamp) {
$dt = new DateTime($timestamp);
return $dt->format('Y-m-d\TH:i:s.v\Z');
}
return null;
}
/**
* Get sync status
*
* @return array Sync status
*/
public function get_status() {
global $wpdb;
$last_sync = $wpdb->get_row(
"SELECT * FROM {$this->db->sync_state_table()}
WHERE status = 'completed'
ORDER BY completed_at DESC
LIMIT 1"
);
$running_sync = $wpdb->get_row(
"SELECT * FROM {$this->db->sync_state_table()}
WHERE status = 'running'
ORDER BY started_at DESC
LIMIT 1"
);
$failed_sync = $wpdb->get_row(
"SELECT * FROM {$this->db->sync_state_table()}
WHERE status = 'failed'
ORDER BY updated_at DESC
LIMIT 1"
);
return array(
'last_sync' => $last_sync,
'running_sync' => $running_sync,
'last_failed' => $failed_sync,
'stats' => $this->db->get_stats(),
);
}
}