Dev.to

Scrape data masjid dengan multi threads

Mencoba untuk melakukan scrape masjid di simas kemenag menggunakan PHP dan multi threads dengan multi curl Scrape akan melakukan scrape masjid id dan lokasi masjid atau koordinatnya di gmaps. Scriptnya:

Apr 10, 2025 - 02:32
 0
Scrape data masjid dengan multi threads

Image description

Mencoba untuk melakukan scrape masjid di simas kemenag menggunakan PHP dan multi threads dengan multi curl

Scrape akan melakukan scrape masjid id dan lokasi masjid atau koordinatnya di gmaps.

Scriptnya:


$startPage   = 1;
$endPage     = 100;
$maxThreads  = 10;
$namaFile    = "hasil_masjid_page{$startPage}-{$endPage}.txt";

// Buka file dan tulis header
$fp = fopen($namaFile, 'w');
fwrite($fp, "\"s_id\",\"link_detil\",\"link_peta\"\n");

// Ambil banyak halaman secara paralel
function fetchPagesMultiCurl($urls) {
    $multiHandle = curl_multi_init();
    $curlHandles = [];
    $results = [];

    foreach ($urls as $key => $url) {
        $ch = curl_init();
        curl_setopt_array($ch, [
            CURLOPT_URL => $url,
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_USERAGENT => 'Mozilla/5.0',
            CURLOPT_TIMEOUT => 10,
        ]);
        curl_multi_add_handle($multiHandle, $ch);
        $curlHandles[$key] = $ch;
    }

    $running = null;
    do {
        curl_multi_exec($multiHandle, $running);
        curl_multi_select($multiHandle, 1.0); // Optimasi I/O blocking
    } while ($running > 0);

    foreach ($curlHandles as $key => $ch) {
        $results[$key] = curl_multi_getcontent($ch);
        curl_multi_remove_handle($multiHandle, $ch);
        curl_close($ch);
    }

    curl_multi_close($multiHandle);
    return $results;
}

// Batch processing
for ($batchStart = $startPage; $batchStart <= $endPage; $batchStart += $maxThreads) {
    $batchEnd = min($batchStart + $maxThreads - 1, $endPage);
    $urls = [];

    for ($i = $batchStart; $i <= $batchEnd; $i++) {
        $urls[$i] = "https://simas.kemenag.go.id/page/profilmasjid/0/0/0/0/0?page=$i";
    }

    printf("
Read More

Tags:

Previous Article

Top SaaS SEO Tools in 2025

Next Article

Exploring the Ethics of Synthetic Media and AI-Generated Content

Related Posts

Automating Cross-Region Replication in AWS S3 Using Lambda Triggers

Automating Cross-Region Replication in AWS S3 Using Lam...

Apr 14, 2025  0

This Week In React #221 : State of React, React Router, React Admin | AI & RN,Gesture Handler | TypeScript, Prettier, esbuild...

This Week In React #221 : State of React, React Router,...

Feb 14, 2025  0

Build a Production-Ready SaaS Application

Build a Production-Ready SaaS Application

Mar 18, 2025  0