Glassdoor 职位数据抓取(by URL)

const axios = require("axios");

// API URL
const API_URL = "https://openapi.coreclaw.com/api/v1/scraper/run";

// Your API KEY
const API_KEY = "<YOUR_API_KEY>";

// Callback URL, The endpoint that will receive the scraping results
const CALLBACK_URL = "https://your-domain.com/callback";

// Build request parameters
const requestBody = {
    "scraper_slug": "01KG2DV66JTCN65ZBTRX3M456E",
    "version": "v1.0.8",
    "input": {
        "parameters": {
            "system": {
                "proxy_region": "",
                "cpus": 0.125,
                "memory": 512,
                "execute_limit_time_seconds": 1800,
                "max_total_charge": 0,
                "max_total_traffic": 0,
            },
            "custom": {
          "url": [
                    {
                              "url": "https://www.glassdoor.com/job-listing/staff-product-analyst-intuit-JV_IC4508422_KO0,21_KE22,28.htm?jl=1008980804695&cs=1_d6b1b697&s=58&t=SR&pos=104&src=GD_JOB_AD&cpc=44CD5376B8534B8F&guid=0000018c0ae77315b0674c403cf266a5&jobListingId=1008980804695&ao=1110586&vt=w&jrtk=5-yul1-0-1hg5eesq3ir3v800-4dda97f9dab7b990---6NYlbfkN0BKYl3zWsktiTMfhnn8eMRsNrNhPXFqHgxANdB9sZgO2gg8MIfiMoKrpC4RrjRFuq5cN1FayxkUzXNpGKe4khcoosOdCsWKdjNUQlszKRvhnVCv-3GxQD0UURkmC2SRM5G4PDS-csRoploh14dWMXKmKSHFODefNWscELEdL9st3xF8QQKpVtPfaY0ycbd-ETjhmsqIBjkCxzKMwE-vSoMbWao2wahtIcpefclgD97rRwH69BkyKCUkw6_dAAZrs2ADvu3bHDeyOTLSIlwBZqRxVe05yri9f7rkilGsITPCgvv3Cfg9tReQUvDznUmY5TAQO1ykaDDCaaDZNjow2bO8henmAFyasHjvhLiubxbzO_JweUKeGcMqwwy80Wgmp5Xcz5rtTc-1vuQXmy3hK-lo-60GgtgLcZbRdKat1Z8tQww1fcL-tfB4nfNQiZmhXUNKNp-K-ChnjJ8FFwNL48I5GC9uunA74XRNacCIRa-mQAw991VyPZoA4xpzc-zfhk12BJvfbAKDgmBIOLa13A9Yw7poZBIKu1yvAsV1a2ejboFEtXzdxtOfmooRjrOTHx67cpRbppk-KmwyuMvjHWJrz_3XSepcuUm7zp9tGBliAyeRX01IPuVMoGsN7CBoTn3zS9kEwLVU6ejV0l1hF5437iDfn61jqnywAzCrmOCaKouMCFJBj2Yage5R4dIdFcm53lB9xSqWy9zGdJ31N5Xj&cb=1700989989992&ctt=1700990085479"
                    }
          ]
}
        }
    },
    "callback_url": CALLBACK_URL
};

async function runScraper() {
    try {
        const response = await axios.post(API_URL, requestBody, {
            headers: {
                "api-key": API_KEY,
                "Content-Type": "application/json"
            },
            timeout: 30000
        });

        const { code, message, data } = response.data

        // Check response status
        if(code === 0) {
            console.log("Status Code:", code)
            console.log("Response Message:", message)
            console.log("Response Body:", JSON.stringify(data))
        } else {
            console.log("Error Status Code:", code)
            console.log("Error Message:", message)
        }
    } catch (error) {
        console.error("Request failed:", error.message);
    }
}

// Execute
runScraper();