daveadmin/dobetternorge-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Exclude dobetternorge.no docs from all corpus search modes

Browse Source 
BM25: adds NOT LIKE filter to SQL WHERE in both FULLTEXT and LIKE paths.
Hybrid + Vector: post-filter hits array by source_url after results return.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
davegilligan
2026-05-15 12:10:46 +02:00
parent 38255669a9
commit 85a6bc8134
1 changed files with 9 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/corpus-search.php
+9 -2
View File
@@ -14,6 +14,8 @@ $language = dbnToolsNormalizeLanguage($input['language'] ?? 'en');
$limit = max(1, min(20, (int)($input['limit'] ?? 8)));
$category = isset($input['category']) && $input['category'] !== '' ? trim((string)$input['category']) : null;
const EXCLUDED_DOMAIN = 'dobetternorge.no';
if (mb_strlen($query, 'UTF-8') < 3) {
dbnToolsError('Query must be at least 3 characters.', 422, 'query_too_short');
}
@@ -33,6 +35,7 @@ try {
'source_url' => $h['source_url'] ?? null,
'language' => null,
], $result['hits'] ?? []);
$hits = array_values(array_filter($hits, fn($h) => !str_contains($h['source_url'] ?? '', EXCLUDED_DOMAIN)));
dbnToolsRespond(['ok' => true, 'hits' => $hits, 'mode' => 'hybrid', 'query' => $query]);
}
@@ -41,6 +44,7 @@ try {
// ── BM25: FULLTEXT with LIKE fallback ───────────────────────────────────
if ($mode === 'bm25') {
$catClause = $category !== null ? ' AND d.category = ?' : '';
$excludeLike = '%' . EXCLUDED_DOMAIN . '%';
// Try FULLTEXT index first
try {
@@ -52,10 +56,11 @@ try {
JOIN documents d ON c.document_id = d.id
WHERE d.corpus_id = ? AND d.status = 'ready'
AND MATCH(c.content) AGAINST (? IN BOOLEAN MODE) > 0
AND d.source_url NOT LIKE ?
$catClause
ORDER BY score DESC
LIMIT ?";
$params = [$query, 1, $query];
$params = [$query, 1, $query, $excludeLike];
if ($category !== null) $params[] = $category;
$params[] = $limit;
$stmt = $ragDb->prepare($sql);
@@ -72,10 +77,11 @@ try {
JOIN documents d ON c.document_id = d.id
WHERE d.corpus_id = ? AND d.status = 'ready'
AND (c.content LIKE ? OR d.title LIKE ?)
AND d.source_url NOT LIKE ?
$catClause
ORDER BY (d.title LIKE ?) DESC
LIMIT ?";
$params = [1, $like, $like];
$params = [1, $like, $like, $excludeLike];
if ($category !== null) $params[] = $category;
$params[] = $like;
$params[] = $limit;
@@ -152,6 +158,7 @@ try {
];
}
$hits = array_values(array_filter($hits, fn($h) => !str_contains($h['source_url'] ?? '', EXCLUDED_DOMAIN)));
dbnToolsRespond(['ok' => true, 'hits' => $hits, 'mode' => 'vector', 'query' => $query]);
}