Glassdoor 职位数据抓取(by URL)

package main

import (
    "bytes"
    "encoding/json"
    "fmt"
    "io"
    "net/http"
    "time"
)

// API URL
const API_URL = "https://openapi.coreclaw.com/api/v1/scraper/run"

// Your API KEY
const API_KEY = "<YOUR_API_KEY>"

// Callback URL, The endpoint that will receive the scraping results
const CALLBACK_URL = "https://your-domain.com/callback"

// ScraperRunRequest represents the structure for running a scraper request
type ScraperRunRequest struct {
    ScraperSlug string          `json:"scraper_slug"` // Unique identifier for the scraper
    Version     string          `json:"version"`      // Worker version number
    Input       json.RawMessage `json:"input"`        // Input parameters
    CallbackURL string          `json:"callback_url"` // Callback URL
}

// ScraperRunResponse represents the structure for the scraper run response
type ScraperRunResponse struct {
    Code    int    `json:"code"`    // Error code
    Message string `json:"message"` // Error message
    Data    Data   `json:"data"`    // Response data
}

// Data represents the structure for response data
type Data struct {
    RunSlug string `json:"run_slug"` // Unique identifier for the run record
}

func main() {
    // Build request parameters
    req := ScraperRunRequest{
        ScraperSlug: "01KG2DV66JTCN65ZBTRX3M456E",
        Version: "v1.0.8",
        Input: json.RawMessage(`{
          "system": {
              "proxy_region": "",
              "cpus": 0.125,
              "memory": 512,
              "execute_limit_time_seconds": 1800,
              "max_total_charge": 0,
              "max_total_traffic": 0
          },
          "custom": {
          "url": [
                    {
                              "url": "https://www.glassdoor.com/job-listing/staff-product-analyst-intuit-JV_IC4508422_KO0,21_KE22,28.htm?jl=1008980804695&cs=1_d6b1b697&s=58&t=SR&pos=104&src=GD_JOB_AD&cpc=44CD5376B8534B8F&guid=0000018c0ae77315b0674c403cf266a5&jobListingId=1008980804695&ao=1110586&vt=w&jrtk=5-yul1-0-1hg5eesq3ir3v800-4dda97f9dab7b990---6NYlbfkN0BKYl3zWsktiTMfhnn8eMRsNrNhPXFqHgxANdB9sZgO2gg8MIfiMoKrpC4RrjRFuq5cN1FayxkUzXNpGKe4khcoosOdCsWKdjNUQlszKRvhnVCv-3GxQD0UURkmC2SRM5G4PDS-csRoploh14dWMXKmKSHFODefNWscELEdL9st3xF8QQKpVtPfaY0ycbd-ETjhmsqIBjkCxzKMwE-vSoMbWao2wahtIcpefclgD97rRwH69BkyKCUkw6_dAAZrs2ADvu3bHDeyOTLSIlwBZqRxVe05yri9f7rkilGsITPCgvv3Cfg9tReQUvDznUmY5TAQO1ykaDDCaaDZNjow2bO8henmAFyasHjvhLiubxbzO_JweUKeGcMqwwy80Wgmp5Xcz5rtTc-1vuQXmy3hK-lo-60GgtgLcZbRdKat1Z8tQww1fcL-tfB4nfNQiZmhXUNKNp-K-ChnjJ8FFwNL48I5GC9uunA74XRNacCIRa-mQAw991VyPZoA4xpzc-zfhk12BJvfbAKDgmBIOLa13A9Yw7poZBIKu1yvAsV1a2ejboFEtXzdxtOfmooRjrOTHx67cpRbppk-KmwyuMvjHWJrz_3XSepcuUm7zp9tGBliAyeRX01IPuVMoGsN7CBoTn3zS9kEwLVU6ejV0l1hF5437iDfn61jqnywAzCrmOCaKouMCFJBj2Yage5R4dIdFcm53lB9xSqWy9zGdJ31N5Xj&cb=1700989989992&ctt=1700990085479"
                    }
          ]
}
        }`),
        CallbackURL: CALLBACK_URL,
    }

    // Send request
    runSlug, err := runScraper(req, API_KEY)
    if err != nil {
        fmt.Printf("Request failed: %v
", err)
        return
    }

    fmt.Printf("Worker run successful!")
    fmt.Printf("Run record ID: %s
", runSlug)
    fmt.Printf("You can use this ID to query run status and results
")
}

// runScraper executes the scraper
func runScraper(req ScraperRunRequest, apiKey string) (string, error) {
    // Serialize request data
    body, err := json.Marshal(req)
    if err != nil {
        return "", fmt.Errorf("failed to serialize request data: %w", err)
    }

    // Create HTTP request
    client := &http.Client{
        Timeout: 30 * time.Second,
    }

    httpReq, err := http.NewRequest(
        "POST",
        API_URL,
        bytes.NewBuffer(body),
    )
    if err != nil {
        return "", fmt.Errorf("failed to create request: %w", err)
    }

    // Set request headers
    httpReq.Header.Set("api-key", apiKey)
    httpReq.Header.Set("Content-Type", "application/json")

    // Send request
    resp, err := client.Do(httpReq)
    if err != nil {
        return "", fmt.Errorf("failed to send request: %w", err)
    }
    defer resp.Body.Close()

    // Read response
    respBody, err := io.ReadAll(resp.Body)
    if err != nil {
        return "", fmt.Errorf("failed to read response: %w", err)
    }

    // Check response status code
    if resp.StatusCode != http.StatusOK {
        return "", fmt.Errorf("request failed, status code: %d, response: %s", resp.StatusCode, string(respBody))
    }

    // Parse response
    var result ScraperRunResponse
    if err := json.Unmarshal(respBody, &result); err != nil {
        return "", fmt.Errorf("failed to parse response: %w", err)
    }

    // Check error code
    if result.Code != 0 {
        return "", fmt.Errorf("business error: %s (error code: %d)", result.Message, result.Code)
    }

    return result.Data.RunSlug, nil
}