brianhansonxyz/homeproz
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Snapshot: MLS sync fixes, image refresh, plugin/theme updates

Browse Source 
MLS plugin fixes from this session:
- Fix silent insert failures: location column NOT NULL was rejecting wpdb->insert calls,
  causing ~18k new properties since Dec 2025 to be lost. Inserts now build raw SQL
  with ST_PointFromText so the spatial column is populated atomically.
- Auto-refresh expired media URLs in MLS_Media_Handler::fetch_and_cache(), guarded by
  a property-level GET_LOCK so concurrent fetches share one API refresh.
- Normalize WP_Error to null in mls_get_property_image() so callers can rely on the
  documented string|null contract.
- Support comma-separated property_type filters in MLS_Query and MLS_Cluster so the
  homepage "View All Commercial" link (?property_type=Commercial+Sale,Land,Farm)
  actually filters correctly.
- Incremental sync now looks back 10 minutes past the latest modification timestamp
  as a safety margin against missed records.
- Smart sync exits silently (info-level, not warning) when a full sync is in progress.

Operational:
- New cron: weekly full sync Sundays at 3 AM (/usr/local/bin/mls-full-sync).
- New cron: hourly 2GB cap on mls-thumbnails/ and cache/transformed-images/
  (/usr/local/bin/mls-image-cache-cap).
- Logrotate config for wp-content/debug.log (2-day retention, daily rotation,
  delaycompress).

Repo policy:
- CLAUDE.md updated with explicit "commit everything except build artifacts" policy.
- .gitignore: untrack runtime image caches and debug.log rotations.

Other modifications in this snapshot are pre-existing in-flight theme/plugin/db_content_updates work.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
root
2026-04-29 15:32:23 +00:00
parent 57b752f54e
commit b6df4dbb92
5385 changed files with 838580 additions and 2416 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wp-content/plugins/mls-by-hansonxyz/includes/class-mls-rate-limiter.php
+149 -39
View File
@@ -29,15 +29,37 @@ class MLS_Rate_Limiter {
const MLSGRID_BYTES_PER_DAY = 42949672960; // 40GB
/**
* Sync operation limits (50% of daily quota paced over 24 hours)
* Sync operation limits
*
* Goal: If sync ran continuously for 24h, use max 50% of daily quota
* - 20,000 requests / 86,400 seconds = 0.23 RPS (~4.3s between requests)
* - 20GB data / 86,400 seconds = ~243KB/s average
* Fixed 5-second interval between API requests for rock-solid rate limiting.
* This ensures we never exceed MLS Grid limits regardless of sync duration.
*
* At 5s intervals: 17,280 requests/day max (43% of 40,000 limit)
*/
const SYNC_REQUESTS_PER_DAY = 20000; // 50% of 40,000
const SYNC_REQUESTS_PER_DAY = 17280; // 86400s / 5s = 17,280 max
const SYNC_BYTES_PER_DAY = 21474836480; // 20GB (50% of 40GB)
const SYNC_MIN_INTERVAL_MS = 4320; // 86400000ms / 20000 = 4.32s between requests
const SYNC_MIN_INTERVAL_MS = 5000; // 5 seconds between requests (legacy)
/**
* Global rate limit intervals (cross-process coordination via MySQL advisory locks)
*
* These are enforced across ALL processes to stay well under MLS Grid limits.
* MLS Grid warns at 2 RPS, suspends at 4+ RPS. We use conservative intervals.
*/
const GENERAL_API_INTERVAL_MS = 4000; // 4 seconds between general API requests
const IMAGE_API_INTERVAL_MS = 2000; // 2 seconds between image API requests
/**
* Advisory lock names for cross-process coordination
*/
const LOCK_GENERAL_API = 'mls_api_general';
const LOCK_IMAGE_API = 'mls_api_image';
/**
* Option keys for storing last request times
*/
const OPTION_LAST_GENERAL_REQUEST = 'mls_last_general_api_request';
const OPTION_LAST_IMAGE_REQUEST = 'mls_last_image_api_request';
/**
* Rate limit constants (used for tracking against MLS Grid limits)
@@ -58,11 +80,6 @@ class MLS_Rate_Limiter {
*/
private $db;
/**
* Last request timestamp for per-second limiting
*/
private $last_request_time = 0;
/**
* Constructor
*
@@ -72,19 +89,131 @@ class MLS_Rate_Limiter {
$this->db = $db;
}
/**
* Wait for and acquire the global API rate limit (general API)
*
* Uses MySQL advisory locks to coordinate across all PHP processes.
* Enforces 4-second minimum interval between general API requests.
*
* @param int $timeout_seconds Max seconds to wait for lock (0 = non-blocking)
* @return bool True if rate limit acquired, false if timeout
*/
public function acquire_general_api_slot($timeout_seconds = 30) {
return $this->acquire_api_slot(
self::LOCK_GENERAL_API,
self::OPTION_LAST_GENERAL_REQUEST,
self::GENERAL_API_INTERVAL_MS,
$timeout_seconds
);
}
/**
* Wait for and acquire the global API rate limit (image API)
*
* Uses MySQL advisory locks to coordinate across all PHP processes.
* Enforces 2-second minimum interval between image API requests.
*
* @param int $timeout_seconds Max seconds to wait for lock (0 = non-blocking)
* @return bool True if rate limit acquired, false if timeout
*/
public function acquire_image_api_slot($timeout_seconds = 30) {
return $this->acquire_api_slot(
self::LOCK_IMAGE_API,
self::OPTION_LAST_IMAGE_REQUEST,
self::IMAGE_API_INTERVAL_MS,
$timeout_seconds
);
}
/**
* Internal method to acquire an API slot with advisory lock coordination
*
* @param string $lock_name Advisory lock name
* @param string $option_key Option key for last request timestamp
* @param int $interval_ms Minimum interval between requests in milliseconds
* @param int $timeout_seconds Max seconds to wait
* @return bool True if slot acquired
*/
private function acquire_api_slot($lock_name, $option_key, $interval_ms, $timeout_seconds) {
global $wpdb;
$start_time = time();
$interval_sec = $interval_ms / 1000.0;
while (true) {
// Check timeout
if ($timeout_seconds > 0 && (time() - $start_time) >= $timeout_seconds) {
return false;
}
// Try to acquire the advisory lock (blocking for up to 1 second)
$lock_acquired = $wpdb->get_var($wpdb->prepare(
"SELECT GET_LOCK(%s, %d)",
$lock_name,
1 // 1 second timeout for each attempt
));
if ($lock_acquired !== '1') {
// Lock held by another process, wait and retry
usleep(100000); // 100ms
continue;
}
try {
// We have the lock - check/wait for rate limit interval
$last_request = (float) get_option($option_key, 0);
$now = microtime(true);
$elapsed = $now - $last_request;
if ($elapsed < $interval_sec) {
// Need to wait for the remaining interval
$wait_time = ($interval_sec - $elapsed) * 1000000; // Convert to microseconds
usleep((int) $wait_time);
}
// Update the last request timestamp
update_option($option_key, microtime(true), false); // false = don't autoload
return true;
} finally {
// Always release the advisory lock
$wpdb->query($wpdb->prepare("SELECT RELEASE_LOCK(%s)", $lock_name));
}
}
}
/**
* Rate limit channels
*/
const CHANNEL_GENERAL = 'general';
const CHANNEL_IMAGE = 'image';
/**
* Check if we can make a request (and wait if needed)
*
* For sync operations, this enforces the 50% daily quota pacing.
* The minimum interval between requests ensures that even continuous
* syncing won't exceed 50% of the daily quota.
* Uses global advisory lock-based rate limiting to coordinate across
* all PHP processes. Different channels have different intervals:
* - general: 4-second interval
* - image: 2-second interval
*
* @param bool $wait Whether to wait if rate limited
* @param string $channel Rate limit channel ('general' or 'image')
* @return bool True if request can proceed
*/
public function check_and_wait($wait = true) {
// Enforce sync pacing (4.32s between requests for 50% daily quota)
$this->enforce_sync_pacing();
public function check_and_wait($wait = true, $channel = self::CHANNEL_GENERAL) {
// Use global advisory lock-based rate limiting
$timeout = $wait ? 60 : 0;
if ($channel === self::CHANNEL_IMAGE) {
if (!$this->acquire_image_api_slot($timeout)) {
return false;
}
} else {
if (!$this->acquire_general_api_slot($timeout)) {
return false;
}
}
// Check hourly limit (hard stop if approaching MLS Grid limits)
if (!$this->check_limit(self::WINDOW_HOUR, self::LIMIT_PER_HOUR)) {
@@ -107,27 +236,6 @@ class MLS_Rate_Limiter {
return true;
}
/**
* Enforce sync operation pacing
*
* Ensures minimum interval between sync requests so that
* 24 hours of continuous syncing uses max 50% of daily quota.
*/
private function enforce_sync_pacing() {
$now = microtime(true);
$min_interval = self::SYNC_MIN_INTERVAL_MS / 1000.0; // Convert ms to seconds (4.32s)
if ($this->last_request_time > 0) {
$elapsed = $now - $this->last_request_time;
if ($elapsed < $min_interval) {
$sleep_time = ($min_interval - $elapsed) * 1000000; // microseconds
usleep((int) $sleep_time);
}
}
$this->last_request_time = microtime(true);
}
/**
* Check if under the limit for a window type
*
@@ -474,6 +582,8 @@ class MLS_Rate_Limiter {
public function reset() {
global $wpdb;
$wpdb->query("TRUNCATE TABLE {$this->db->rate_limits_table()}");
$this->last_request_time = 0;
// Reset global timestamps
delete_option(self::OPTION_LAST_GENERAL_REQUEST);
delete_option(self::OPTION_LAST_IMAGE_REQUEST);
}
}