>>11113665oh yk why? I forgot I did some work on the script to make it more efficient and split everything into its own table, which used to not be a thing. It was all in one table initially.
>>11113928It doesn't. Here's the relevant code on scrape.php:
foreach (array_keys(BOARDS) as $board) {
error_log("Scraping board /".$board."/");
for ($page = 0; $page < PAGES; $page++) {
error_log(" Scraping page ".$page);
$json = @file_get_contents(VICHAN_URL."/".$board."/".$page.".json");
if ($json === FALSE)
continue;
$data = json_decode($json);
foreach ($data->threads as $thread) {
if ($thread->posts[0]->last_modified < $lastScrape)
continue;
error_log(" Scraping thread ".$thread->posts[0]->no);
$tjson = @file_get_contents(VICHAN_URL."/".$board."/thread/".$thread->posts[0]->no.".json");
if ($tjson === FALSE)
continue;
$tdata = json_decode($tjson);
foreach ($tdata->posts as $post) {
if ($post->last_modified < $lastScrape)
continue;
$exst = $conn->prepare("SELECT * FROM archive WHERE board = ? AND postnum = ?");
$exst->bind_param("si", $board, $post->no);
$exst->execute();
$res = $exst->get_result();
if ($res->num_rows)
continue;
error_log(" Post ".$post->no);
/* Handle multiple files */
$filenames = array();
$exts = array();
$tims = array();
if (isset($post->filename) && $post->filename) {
$filenames[] = $post->filename;
$exts[] = $post->ext;
$tims[] = $post->tim;
}
if (isset($post->extra_files) && $post->extra_files) {
foreach ($post->extra_files as $extra) {
$filenames[] = $extra->filename;
$exts[] = $extra->ext;
$tims[] = $extra->tim;
}
}
$filename = implode("/",$filenames);
$ext = implode("/",$exts);
if ($post->resto) {
$stmt = $conn->prepare("INSERT INTO archive (thread, subject, email, name, trip, capcode, body, time, filename, ext, board, postnum) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)");
$stmt->bind_param("issssssisssi", $post->resto, $post->sub, $post->email, $post->name, $post->trip, $post->capcode, $post->com, $post->time, $filename, $ext, $board, $post->no);
} else {
$stmt = $conn->prepare("INSERT INTO archive (subject, email, name, trip, capcode, body, time, filename, ext, replies, images, board, postnum) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)");
$stmt->bind_param("ssssssissiisi", $post->sub, $post->email, $post->name, $post->trip, $post->capcode, $post->com, $post->time, $filename, $ext, $thread->posts[0]->replies, $thread->posts[0]->images, $board, $post->no);
}
$stmt->execute();
$id = $stmt->insert_id;
share
download .zip
report bug or abuse
Buy me a coffee
