Jedná sa o vyhľadávanie v databáze, nie scrapping, čo je podľa zmlúv dovolené: https://hrubos.tech/blogy/content/images/20260911131807-nespcarpping.png
https://hrubos.tech/blogy/content/images/20260911125802-prometheus.png
https://hrubos.tech/blogy/content/images/20260911125947-prometheus2.png
https://hrubos.tech/blogy/content/images/20260911130020-prometheus3.png
https://hrubos.tech/blogy/content/images/20260911130057-prometheus4.png
Ochutnávka: tail -f storage/logs/lidl.log
á do /config/logging.php dať:
'lidl' => [
'driver' => 'single',
'path' => storage_path('logs/lidl.log'),
'level' => 'info',
'replace_placeholders' => true,
],
'kaufland' => [
'driver' => 'single',
'path' => storage_path('logs/kaufland.log'),
'level' => 'info',
'replace_placeholders' => true,
],
Potom klasika kontrolér:
lesnavevericka@192 projects % curl -sS \
'http://localhost:8000/api/interconnector/lidl'
{"success":true,"shop":"lidl","scraped_at":"2026-09-11T10:57:26+00:00","search_terms":1602,"successful_searches":1158,"failed_searches":444,"raw_products":11394,"product_count":1012,"duplicates_removed":10382,"duration_seconds":1944.53,"file":"private\/prometheus\/lidl.json"}% lesnavevericka@192 projects %
Náhľad jedného z kontrollérov:
<?php
namespace App\Http\Controllers\Interconnector;
use Illuminate\Support\Facades\Http;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Storage;
use Throwable;
class LidlController
{
private const SEARCH_URL = 'https://www.lidl.sk/q/search';
private const CSV_FILE = 'private/prometheus/potraviny.csv';
private const OUTPUT_FILE = 'private/prometheus/lidl.json';
private const PROGRESS_FILE =
'private/prometheus/lidl-progress.json';
private const USER_AGENT =
'Mozilla/5.0 (X11; Linux x86_64; rv:142.0) '
. 'Gecko/20100101 Firefox/142.0';
/**
* Koľko výrazov spracovať medzi checkpointmi.
*/
private const CHECKPOINT_EVERY = 25;
public function index()
{
/*
* PHP nesmie ukončiť script kvôli execution timeoutu.
*/
ini_set('memory_limit', '4G');
set_time_limit(0);
$startedAt = microtime(true);
Log::channel('lidl')->info(
'========== LIDL RUN START =========='
);
try {
/*
* -----------------------------------------------------
* TEST JEDNÉHO VÝRAZU
* -----------------------------------------------------
*
* Napr.:
*
* /api/interconnector/lidl?q=ovocny+jogurt
*
* Tento režim nezasahuje do checkpointu ani lidl.json.
*/
$testTerm = request()->query('q');
if (
is_string($testTerm) &&
trim($testTerm) !== ''
) {
$testTerm = trim($testTerm);
Log::channel('lidl')->info(
'[Lidl] TEST REŽIM: "' . $testTerm . '"'
);
$found = $this->searchWebPage($testTerm);
Log::channel('lidl')->info(
'[Lidl] TEST HOTOVÝ: ' .
count($found) .
' produktov.'
);
return response()->json([
'success' => true,
'test' => true,
'shop' => 'lidl',
'search_term' => $testTerm,
'product_count' => count($found),
'products' => $found,
'duration_seconds' =>
round(
microtime(true) - $startedAt,
2
),
]);
}
/*
* -----------------------------------------------------
* RESET
* -----------------------------------------------------
*
* /api/interconnector/lidl?reset=1
*
* Vymaže checkpoint a začne od prvého výrazu.
*/
$reset = request()->boolean('reset');
if ($reset) {
$this->deleteProgress();
Log::channel('lidl')->info(
'[Lidl] RESET: checkpoint bol vymazaný.'
);
}
/*
* -----------------------------------------------------
* CSV
* -----------------------------------------------------
*/
$terms = $this->loadSearchTerms();
$total = count($terms);
Log::channel('lidl')->info(
'[Lidl] Načítaných vyhľadávacích výrazov: ' .
$total
);
if ($total === 0) {
throw new \RuntimeException(
'CSV neobsahuje žiadne vyhľadávacie výrazy.'
);
}
/*
* -----------------------------------------------------
* OBNOVENIE CHECKPOINTU
* -----------------------------------------------------
*/
$progress = $this->loadProgress();
$startIndex = 0;
$products = [];
$successfulSearches = 0;
$failedSearches = 0;
$rawProducts = 0;
if ($progress !== null) {
/*
* Kontrolujeme, či checkpoint patrí rovnakému
* počtu výrazov.
*/
if (
isset($progress['total_terms']) &&
(int) $progress['total_terms'] === $total
) {
$startIndex =
max(
0,
min(
$total,
(int) (
$progress['next_index']
?? 0
)
)
);
$successfulSearches =
(int) (
$progress['successful_searches']
?? 0
);
$failedSearches =
(int) (
$progress['failed_searches']
?? 0
);
$rawProducts =
(int) (
$progress['raw_products']
?? 0
);
/*
* Obnovíme produkty.
*/
if (
isset($progress['products']) &&
is_array($progress['products'])
) {
foreach (
$progress['products']
as $product
) {
if (
!is_array($product) ||
!isset(
$product['product_id']
)
) {
continue;
}
$products[
(string)
$product['product_id']
] = $product;
}
}
Log::channel('lidl')->info(
sprintf(
'[Lidl] OBNOVENIE: pokračujem od %d/%d. ' .
'Doteraz unikátnych produktov: %d, ' .
'úspešných: %d, chybných: %d.',
$startIndex,
$total,
count($products),
$successfulSearches,
$failedSearches
)
);
} else {
Log::channel('lidl')->warning(
'[Lidl] Checkpoint má iný počet výrazov. ' .
'Začínam od nuly.'
);
}
} else {
Log::channel('lidl')->info(
'[Lidl] Checkpoint neexistuje. ' .
'Začínam od prvého výrazu.'
);
}
/*
* -----------------------------------------------------
* HLAVNÝ LOOP
* -----------------------------------------------------
*/
for (
$index = $startIndex;
$index < $total;
$index++
) {
$term = $terms[$index];
$number = $index + 1;
$percent = (int) floor(
($number / $total) * 100
);
Log::channel('lidl')->info(
sprintf(
'[Lidl] [%d/%d] %d%% - "%s"',
$number,
$total,
$percent,
$term
)
);
$searchStarted = microtime(true);
try {
$found = $this->searchWebPage($term);
$count = count($found);
$rawProducts += $count;
$successfulSearches++;
foreach ($found as $product) {
$productId =
$product['product_id']
?? null;
if ($productId === null) {
continue;
}
$productId = (string) $productId;
/*
* Ak produkt už existuje, zachováme jeho
* existujúce search_terms a pridáme nový.
*/
if (
isset(
$products[$productId]
)
) {
$existing =
$products[$productId];
$existingTerms =
$existing['search_terms']
?? [];
if (
!in_array(
$term,
$existingTerms,
true
)
) {
$existingTerms[] = $term;
}
$product['search_terms'] =
$existingTerms;
$products[$productId] =
$product;
} else {
$products[$productId] =
$product;
}
}
$elapsed =
microtime(true)
- $searchStarted;
Log::channel('lidl')->info(
sprintf(
'[Lidl] OK: %d produktov ' .
'(%.2fs), spolu: %d',
$count,
$elapsed,
count($products)
)
);
} catch (Throwable $e) {
$failedSearches++;
Log::channel('lidl')->error(
'[Lidl] EXCEPTION: ' .
$e->getMessage()
);
}
/*
* -------------------------------------------------
* CHECKPOINT
* -------------------------------------------------
*
* Uložíme stav po každom 25. výraze.
*
* Ak webserver spojenie zabije, máme maximálne
* posledných 24 výrazov, o ktoré môžeme prísť.
*/
$completed = $index + 1;
if (
$completed % self::CHECKPOINT_EVERY === 0 ||
$completed === $total
) {
$this->saveProgress([
'success' => true,
'shop' => 'lidl',
'updated_at' =>
now()->toIso8601String(),
'total_terms' => $total,
'completed_terms' =>
$completed,
'next_index' =>
$completed,
'successful_searches' =>
$successfulSearches,
'failed_searches' =>
$failedSearches,
'raw_products' =>
$rawProducts,
'product_count' =>
count($products),
'products' =>
array_values($products),
]);
Log::channel('lidl')->info(
sprintf(
'[Lidl] CHECKPOINT: %d/%d ' .
'(%d%%), produkty: %d',
$completed,
$total,
(int) floor(
($completed / $total) * 100
),
count($products)
)
);
}
/*
* Pauza medzi Lidl requestami.
*/
usleep(200000);
}
/*
* -----------------------------------------------------
* CELÉ KOLO HOTOVÉ
* -----------------------------------------------------
*/
$products = array_values($products);
/*
* Ochrana proti zlému parseru.
*/
if (count($products) === 0) {
throw new \RuntimeException(
'Lidl parser returned 0 products.'
);
}
/*
* Ak máme extrémne veľa chýb a nulové výsledky
* z väčšiny requestov, radšej JSON neprepíšeme.
*/
if (
$successfulSearches === 0
) {
throw new \RuntimeException(
'Lidl nemal ani jedno úspešné vyhľadávanie.'
);
}
/*
* -----------------------------------------------------
* FINÁLNY JSON
* -----------------------------------------------------
*/
$json = [
'success' => true,
'shop' => 'lidl',
'scraped_at' =>
now()->toIso8601String(),
'search_terms' =>
$total,
'successful_searches' =>
$successfulSearches,
'failed_searches' =>
$failedSearches,
'raw_products' =>
$rawProducts,
'product_count' =>
count($products),
'duplicates_removed' =>
max(
0,
$rawProducts
- count($products)
),
'duration_seconds' =>
round(
microtime(true)
- $startedAt,
2
),
'products' =>
$products,
];
/*
* -----------------------------------------------------
* ATOMICKÝ ZÁPIS
* -----------------------------------------------------
*
* Najprv zapíšeme temporary súbor.
* Až keď je kompletný, nahradíme lidl.json.
*/
$jsonContent = json_encode(
$json,
JSON_PRETTY_PRINT |
JSON_UNESCAPED_UNICODE |
JSON_UNESCAPED_SLASHES |
JSON_THROW_ON_ERROR
);
$temporaryFile =
self::OUTPUT_FILE . '.tmp';
Storage::disk('local')->put(
$temporaryFile,
$jsonContent
);
/*
* Kontrola, že temporary JSON naozaj existuje
* a nie je prázdny.
*/
if (
!Storage::disk('local')->exists(
$temporaryFile
)
) {
throw new \RuntimeException(
'Temporary Lidl JSON sa nepodarilo vytvoriť.'
);
}
$savedContent =
Storage::disk('local')->get(
$temporaryFile
);
if (
trim($savedContent) === ''
) {
throw new \RuntimeException(
'Temporary Lidl JSON je prázdny.'
);
}
/*
* Až teraz nahradíme ostrý JSON.
*/
Storage::disk('local')->move(
$temporaryFile,
self::OUTPUT_FILE
);
/*
* -----------------------------------------------------
* CHECKPOINT UŽ NETREBA
* -----------------------------------------------------
*/
$this->deleteProgress();
Log::channel('lidl')->info(
sprintf(
'[Lidl] HOTOVO: %d unikátnych produktov.',
count($products)
)
);
Log::channel('lidl')->info(
sprintf(
'[Lidl] Úspešné vyhľadávania: %d, ' .
'chybné: %d.',
$successfulSearches,
$failedSearches
)
);
Log::channel('lidl')->info(
'[Lidl] JSON uložený do: ' .
self::OUTPUT_FILE
);
Log::channel('lidl')->info(
'========== LIDL RUN END =========='
);
return response()->json([
'success' => true,
'shop' => 'lidl',
'scraped_at' =>
now()->toIso8601String(),
'search_terms' =>
$total,
'successful_searches' =>
$successfulSearches,
'failed_searches' =>
$failedSearches,
'raw_products' =>
$rawProducts,
'product_count' =>
count($products),
'duplicates_removed' =>
max(
0,
$rawProducts
- count($products)
),
'duration_seconds' =>
round(
microtime(true)
- $startedAt,
2
),
'file' =>
self::OUTPUT_FILE,
]);
} catch (Throwable $e) {
Log::channel('lidl')->error(
'[Lidl] FATAL: ' .
$e->getMessage()
);
/*
* Skontrolujeme, či existuje starý JSON.
*/
$oldJsonExists =
Storage::disk('local')->exists(
self::OUTPUT_FILE
);
Log::channel('lidl')->error(
'[Lidl] Existujúci JSON zachovaný: ' .
(
$oldJsonExists
? 'ÁNO'
: 'NIE'
)
);
Log::channel('lidl')->info(
'========== LIDL RUN FAILED =========='
);
/*
* HTTP 500 je zámerný.
*
* Cron tak vie, že beh nedopadol kompletne.
*/
return response()->json([
'success' => false,
'shop' => 'lidl',
'error' =>
$e->getMessage(),
'output_file' =>
self::OUTPUT_FILE,
'existing_json_preserved' =>
$oldJsonExists,
], 500);
}
}
/**
* -------------------------------------------------------------
* CSV
* -------------------------------------------------------------
*/
private function loadSearchTerms(): array
{
$path = storage_path(
'app/' . self::CSV_FILE
);
if (!is_file($path)) {
throw new \RuntimeException(
'CSV súbor neexistuje: ' . $path
);
}
$content = file_get_contents($path);
if ($content === false) {
throw new \RuntimeException(
'Nepodarilo sa načítať CSV.'
);
}
/*
* Odstránenie UTF-8 BOM.
*/
$content = preg_replace(
'/^\xEF\xBB\xBF/',
'',
$content
);
$lines = preg_split(
'/\r\n|\r|\n/',
$content
);
$terms = [];
foreach ($lines as $line) {
$line = trim($line);
if ($line === '') {
continue;
}
$values = str_getcsv(
$line,
';'
);
foreach ($values as $value) {
$value = trim($value);
if ($value !== '') {
$terms[] = $value;
}
}
}
return array_values(
array_unique($terms)
);
}
/**
* -------------------------------------------------------------
* LIDL SEARCH
* -------------------------------------------------------------
*/
private function searchWebPage(
string $term
): array {
$url = self::SEARCH_URL . '?' .
http_build_query(
[
'q' => $term,
'category' =>
'Jedlo a nápoje',
],
'',
'&',
PHP_QUERY_RFC3986
);
$response = Http::withHeaders([
'User-Agent' =>
self::USER_AGENT,
'Accept' =>
'text/html,application/xhtml+xml,'
. 'application/xml;q=0.9,'
. 'image/avif,image/webp,*/*;q=0.8',
'Accept-Language' =>
'sk-SK,sk;q=0.9,en;q=0.8',
'Cache-Control' =>
'no-cache',
'Pragma' =>
'no-cache',
])
->connectTimeout(20)
->timeout(60)
->get($url);
if (!$response->successful()) {
throw new \RuntimeException(
'HTTP ' .
$response->status() .
' pri vyhľadávaní "' .
$term .
'"'
);
}
$html = $response->body();
if (trim($html) === '') {
throw new \RuntimeException(
'Lidl vrátil prázdne HTML.'
);
}
return $this->parseProductsFromHtml(
$html,
$term
);
}
/**
* -------------------------------------------------------------
* HTML PARSER
* -------------------------------------------------------------
*/
private function parseProductsFromHtml(
string $html,
string $searchTerm
): array {
$products = [];
// Lidl HTML obsahuje JSON zapísaný ako HTML entity (" atď.).
$html = html_entity_decode(
$html,
ENT_QUOTES | ENT_HTML5,
'UTF-8'
);
$needle = '"productId":';
$offset = 0;
$length = strlen($html);
while (($position = strpos($html, $needle, $offset)) !== false) {
$offset = $position + strlen($needle);
// Produktové ID musí byť číslo.
$idStart = $position + strlen($needle);
while (
$idStart < $length &&
ctype_space($html[$idStart])
) {
$idStart++;
}
$idEnd = $idStart;
while (
$idEnd < $length &&
ctype_digit($html[$idEnd])
) {
$idEnd++;
}
if ($idEnd === $idStart) {
continue;
}
$productId = substr(
$html,
$idStart,
$idEnd - $idStart
);
// Nájdeme začiatok JSON objektu, v ktorom je productId.
$start = $this->findJsonObjectStart(
$html,
$position
);
if ($start === null) {
continue;
}
// Nájdeme koniec JSON objektu.
$json = $this->extractJsonObject(
$html,
$start
);
if ($json === null) {
continue;
}
$data = json_decode($json, true);
if (
!is_array($data) ||
!isset($data['productId']) ||
!isset($data['fullTitle'])
) {
continue;
}
// Bezpečnostná kontrola.
if ((string) $data['productId'] !== $productId) {
continue;
}
$product = $this->normalizeProduct(
$data,
$searchTerm
);
if ($product === null) {
continue;
}
$products[(string) $product['product_id']] = $product;
}
return array_values($products);
}
private function findJsonObjectStart(
string $text,
int $position
): ?int {
$depth = 0;
$inString = false;
$escaped = false;
for ($i = $position - 1; $i >= 0; $i--) {
$char = $text[$i];
if ($char === '"' && !$escaped) {
$inString = !$inString;
}
if ($inString) {
$escaped = (
$char === '\\' &&
!$escaped
);
continue;
}
if ($char === '}') {
$depth++;
} elseif ($char === '{') {
if ($depth === 0) {
return $i;
}
$depth--;
}
$escaped = false;
}
return null;
}
private function extractJsonObject(
string $text,
int $start
): ?string {
$length = strlen($text);
$depth = 0;
$inString = false;
$escaped = false;
for ($i = $start; $i < $length; $i++) {
$char = $text[$i];
if ($inString) {
if ($char === '"' && !$escaped) {
$inString = false;
}
$escaped = (
$char === '\\' &&
!$escaped
);
continue;
}
if ($char === '"') {
$inString = true;
$escaped = false;
continue;
}
if ($char === '{') {
$depth++;
} elseif ($char === '}') {
$depth--;
if ($depth === 0) {
return substr(
$text,
$start,
$i - $start + 1
);
}
}
}
return null;
}
/**
* -------------------------------------------------------------
* NORMALIZE PRODUCT
* -------------------------------------------------------------
*/
private function normalizeProduct(
array $data,
string $searchTerm
): ?array {
if (
!isset($data['productId']) ||
!isset($data['fullTitle'])
) {
return null;
}
$productId =
(string) $data['productId'];
if ($productId === '') {
return null;
}
$price = null;
$currency = null;
$packaging = null;
$basePrice = null;
$discount = null;
if (
isset($data['price']) &&
is_array($data['price'])
) {
$price =
$data['price']['price']
?? null;
$currency =
$data['price']['currencyCode']
?? null;
$packaging =
$data['price']
['packaging']
['text']
?? null;
$basePrice =
$data['price']
['basePrice']
['text']
?? null;
}
/*
* Regionálna cena.
*/
if (
isset($data['regionsPrices']) &&
is_array(
$data['regionsPrices']
)
) {
foreach (
$data['regionsPrices']
as $region
) {
if (
!isset(
$region['currentPrice']
)
) {
continue;
}
$current =
$region['currentPrice'];
if (
isset($current['price'])
) {
$price =
$current['price'];
}
if (
isset(
$current['currencyCode']
)
) {
$currency =
$current['currencyCode'];
}
if (
isset(
$current['packaging']['text']
)
) {
$packaging =
$current['packaging']['text'];
}
if (
isset(
$current['basePrice']['text']
)
) {
$basePrice =
$current['basePrice']['text'];
}
if (
isset(
$current['discount']
)
) {
$discount =
$current['discount'];
}
break;
}
}
$brand = null;
if (
isset($data['brand']['name'])
) {
$brand =
$data['brand']['name'];
}
$canonicalUrl =
$data['canonicalUrl']
?? $data['canonicalPath']
?? null;
if (
$canonicalUrl !== null &&
!str_starts_with(
$canonicalUrl,
'http'
)
) {
$canonicalUrl =
'https://www.lidl.sk/' .
ltrim(
$canonicalUrl,
'/'
);
}
return [
'product_id' =>
$productId,
'name' =>
trim(
$data['fullTitle']
),
'brand' =>
$brand,
'price' =>
$price,
'currency' =>
$currency,
'packaging' =>
$packaging,
'base_price' =>
$basePrice,
'discount' =>
$discount,
'url' =>
$canonicalUrl,
'image' =>
$data['image']
?? null,
'nat' =>
$data['nat']
?? null,
'category' =>
$data['category']
?? null,
'product_type' =>
$data['productType']
?? null,
'online' =>
$data['online']
?? null,
'regions' =>
$data['regions']
?? [],
'lidl_plus' =>
$data['lidlPlus']
?? [],
'search_terms' => [
$searchTerm,
],
'source' =>
'lidl.sk',
];
}
/**
* -------------------------------------------------------------
* DOM FALLBACK
* -------------------------------------------------------------
*/
private function parseProductsFromDom(
string $html,
string $searchTerm
): array {
$products = [];
$dom =
new \DOMDocument();
libxml_use_internal_errors(true);
$dom->loadHTML(
'<?xml encoding="UTF-8">' .
$html
);
libxml_clear_errors();
$xpath =
new \DOMXPath($dom);
$links =
$xpath->query(
'//a[contains(@href, "/p/")]'
);
if ($links === false) {
return [];
}
foreach ($links as $link) {
$href =
$link->getAttribute('href');
if (
!preg_match(
'~/p/[^/]+/p(\d+)~',
$href,
$match
)
) {
continue;
}
$productId =
$match[1];
$name =
trim(
preg_replace(
'/\s+/',
' ',
$link->textContent
)
);
if ($name === '') {
continue;
}
if (
!str_starts_with(
$href,
'http'
)
) {
$href =
'https://www.lidl.sk/' .
ltrim(
$href,
'/'
);
}
$products[$productId] = [
'product_id' =>
$productId,
'name' =>
$name,
'brand' =>
null,
'price' =>
null,
'currency' =>
'EUR',
'packaging' =>
null,
'base_price' =>
null,
'discount' =>
null,
'url' =>
$href,
'image' =>
null,
'nat' =>
null,
'category' =>
'Food',
'product_type' =>
null,
'online' =>
null,
'regions' =>
[],
'lidl_plus' =>
[],
'search_terms' => [
$searchTerm,
],
'source' =>
'lidl.sk',
];
}
return array_values(
$products
);
}
/**
* -------------------------------------------------------------
* PROGRESS
* -------------------------------------------------------------
*/
private function loadProgress(): ?array
{
if (
!Storage::disk('local')->exists(
self::PROGRESS_FILE
)
) {
return null;
}
try {
$content =
Storage::disk('local')->get(
self::PROGRESS_FILE
);
if (
trim($content) === ''
) {
return null;
}
$data =
json_decode(
$content,
true,
512,
JSON_THROW_ON_ERROR
);
return is_array($data)
? $data
: null;
} catch (Throwable $e) {
Log::channel('lidl')->warning(
'[Lidl] Nepodarilo sa načítať ' .
'checkpoint: ' .
$e->getMessage()
);
return null;
}
}
/**
* Uloží checkpoint.
*/
private function saveProgress(
array $progress
): void {
$content =
json_encode(
$progress,
JSON_PRETTY_PRINT |
JSON_UNESCAPED_UNICODE |
JSON_UNESCAPED_SLASHES |
JSON_THROW_ON_ERROR
);
Storage::disk('local')->put(
self::PROGRESS_FILE,
$content
);
}
/**
* Vymaže checkpoint.
*/
private function deleteProgress(): void
{
if (
Storage::disk('local')->exists(
self::PROGRESS_FILE
)
) {
Storage::disk('local')->delete(
self::PROGRESS_FILE
);
}
}
}
