网页爬虫工具

<?php

// API URL
const API_URL = "https://openapi.coreclaw.com/api/v1/scraper/run";

// Your API KEY
const API_KEY = "<YOUR_API_KEY>";

// curl timeout (seconds)
const TIMEOUT = 30;

/**
 * Run scraper
 *
 * @param array $params Request parameters
 * @param string $apiKey API Key
 * @return array Return result ["success" => bool, "run_slug" => string|null, "error" => string|null]
 */
function runScraper(array $params, string $apiKey): array
{
    // Initialize cURL
    $ch = curl_init();

    // Set cURL options
    curl_setopt_array($ch, [
        CURLOPT_URL => API_URL,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_ENCODING => "",
        CURLOPT_MAXREDIRS => 10,
        CURLOPT_TIMEOUT => TIMEOUT,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
        CURLOPT_CUSTOMREQUEST => "POST",
        CURLOPT_POSTFIELDS => json_encode($params),
        CURLOPT_HTTPHEADER => [
            "api-key: " . $apiKey,
            "Content-Type: application/json"
        ],
    ]);

    // Execute request
    $response = curl_exec($ch);
    $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
    $error = curl_error($ch);

    // Close cURL
    curl_close($ch);

    // Check cURL error
    if ($error) {
        return [
            "success" => false,
            "run_slug" => null,
            "error" => "cURL error: " . $error
        ];
    }

    // Check HTTP status code
    if ($httpCode !== 200) {
        return [
            "success" => false,
            "run_slug" => null,
            "error" => "HTTP error: " . $httpCode . " - " . $response
        ];
    }

    // Parse response
    $result = json_decode($response, true);
    if (json_last_error() !== JSON_ERROR_NONE) {
        return [
            "success" => false,
            "run_slug" => null,
            "error" => "JSON decode error: " . json_last_error_msg()
        ];
    }

    // Check business error code
    if (isset($result["code"]) && $result["code"] !== 0) {
        return [
            "success" => false,
            "run_slug" => null,
            "error" => "Business error: " . (isset($result["message"]) ? $result["message"] : "Unknown error") . " (code: " . $result["code"] . ")"
        ];
    }

    // Return success result
    return [
        "success" => true,
        "run_slug" => isset($result["data"]["run_slug"]) ? $result["data"]["run_slug"] : null,
        "error" => null
    ];
}

/**
 * Main function
 */
function main()
{
    // Build request parameters
    $requestParams = [
        "scraper_slug" => "01KG2DV66JTCN65ZBTRX3M456E",
        "version" => "v1.0.8",
        "input" => [
            "parameters" => [
                "system" => [
                    "proxy_region" => "",
                    "cpus" => 0.125,
                    "memory" => 512,
                    "execute_limit_time_seconds" => 1800,
                    "max_total_charge" => 0,
                    "max_total_traffic" => 0
                ],
                "custom" => {
          'url': [
                    {
                              'url': 'https://example.com'
                    }
          ],
          'maxCrawlingDepth': 1,
          'maxPagesPerCrawl': 10,
          'pageLoadTimeoutSecs': 60,
          'waitUntil': 'networkidle2',
          'injectJQuery': false,
          'ignoreSslErrors': true,
          'downloadMedia': false,
          'downloadCss': true,
          'debugLog': false
}
            ]
        ],
        "callback_url" => "https://your-domain.com/callback"
    ];

    // Send request
    echo "Sending request to API...
";
    $result = runScraper($requestParams, API_KEY);

    // Handle result
    if ($result["success"]) {
        echo "Worker run successful!
";
        echo "Run record ID: " . $result["run_slug"] . "
";
        echo "You can use this ID to query run status and results
";
    } else {
        echo "Request failed!
";
        echo "Error message: " . $result["error"] . "
";
    }
}

// Execute main function
main();