Mon Oct 3 13:36:35 2016 UTC ()
Mark the section and md5_hash columns as unindexed in the FTS table, as they are not used for search


(abhinav)
diff -r1.28 -r1.29 src/usr.sbin/makemandb/apropos-utils.c

cvs diff -r1.28 -r1.29 src/usr.sbin/makemandb/apropos-utils.c (switch to unified diff)

--- src/usr.sbin/makemandb/apropos-utils.c 2016/07/06 18:03:27 1.28
+++ src/usr.sbin/makemandb/apropos-utils.c 2016/10/03 13:36:35 1.29
@@ -1,1091 +1,1092 @@ @@ -1,1091 +1,1092 @@
1/* $NetBSD: apropos-utils.c,v 1.28 2016/07/06 18:03:27 abhinav Exp $ */ 1/* $NetBSD: apropos-utils.c,v 1.29 2016/10/03 13:36:35 abhinav Exp $ */
2/*- 2/*-
3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com> 3 * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
4 * All rights reserved. 4 * All rights reserved.
5 * 5 *
6 * This code was developed as part of Google's Summer of Code 2011 program. 6 * This code was developed as part of Google's Summer of Code 2011 program.
7 * 7 *
8 * Redistribution and use in source and binary forms, with or without 8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions 9 * modification, are permitted provided that the following conditions
10 * are met: 10 * are met:
11 * 11 *
12 * 1. Redistributions of source code must retain the above copyright 12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer. 13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright 14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in 15 * notice, this list of conditions and the following disclaimer in
16 * the documentation and/or other materials provided with the 16 * the documentation and/or other materials provided with the
17 * distribution. 17 * distribution.
18 * 18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 21 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 22 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 28 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE. 30 * SUCH DAMAGE.
31 */ 31 */
32 32
33#include <sys/cdefs.h> 33#include <sys/cdefs.h>
34__RCSID("$NetBSD: apropos-utils.c,v 1.28 2016/07/06 18:03:27 abhinav Exp $"); 34__RCSID("$NetBSD: apropos-utils.c,v 1.29 2016/10/03 13:36:35 abhinav Exp $");
35 35
36#include <sys/queue.h> 36#include <sys/queue.h>
37#include <sys/stat.h> 37#include <sys/stat.h>
38 38
39#include <assert.h> 39#include <assert.h>
40#include <ctype.h> 40#include <ctype.h>
41#include <err.h> 41#include <err.h>
42#include <math.h> 42#include <math.h>
43#include <stdio.h> 43#include <stdio.h>
44#include <stdlib.h> 44#include <stdlib.h>
45#include <string.h> 45#include <string.h>
46#include <util.h> 46#include <util.h>
47#include <zlib.h> 47#include <zlib.h>
48#include <term.h> 48#include <term.h>
49#undef tab // XXX: manconf.h 49#undef tab // XXX: manconf.h
50 50
51#include "apropos-utils.h" 51#include "apropos-utils.h"
52#include "manconf.h" 52#include "manconf.h"
53 53
54typedef struct orig_callback_data { 54typedef struct orig_callback_data {
55 void *data; 55 void *data;
56 int (*callback) (void *, const char *, const char *, const char *, 56 int (*callback) (void *, const char *, const char *, const char *,
57 const char *, size_t); 57 const char *, size_t);
58} orig_callback_data; 58} orig_callback_data;
59 59
60typedef struct inverse_document_frequency { 60typedef struct inverse_document_frequency {
61 double value; 61 double value;
62 int status; 62 int status;
63} inverse_document_frequency; 63} inverse_document_frequency;
64 64
65/* weights for individual columns */ 65/* weights for individual columns */
66static const double col_weights[] = { 66static const double col_weights[] = {
67 2.0, // NAME 67 2.0, // NAME
68 2.00, // Name-description 68 2.00, // Name-description
69 0.55, // DESCRIPTION 69 0.55, // DESCRIPTION
70 0.10, // LIBRARY 70 0.10, // LIBRARY
71 0.001, //RETURN VALUES 71 0.001, //RETURN VALUES
72 0.20, //ENVIRONMENT 72 0.20, //ENVIRONMENT
73 0.01, //FILES 73 0.01, //FILES
74 0.001, //EXIT STATUS 74 0.001, //EXIT STATUS
75 2.00, //DIAGNOSTICS 75 2.00, //DIAGNOSTICS
76 0.05, //ERRORS 76 0.05, //ERRORS
77 0.00, //md5_hash 77 0.00, //md5_hash
78 1.00 //machine 78 1.00 //machine
79}; 79};
80 80
81/* 81/*
82 * lower -- 82 * lower --
83 * Converts the string str to lower case 83 * Converts the string str to lower case
84 */ 84 */
85char * 85char *
86lower(char *str) 86lower(char *str)
87{ 87{
88 assert(str); 88 assert(str);
89 int i = 0; 89 int i = 0;
90 char c; 90 char c;
91 while (str[i] != '\0') { 91 while (str[i] != '\0') {
92 c = tolower((unsigned char) str[i]); 92 c = tolower((unsigned char) str[i]);
93 str[i++] = c; 93 str[i++] = c;
94 } 94 }
95 return str; 95 return str;
96} 96}
97 97
98/* 98/*
99* concat-- 99* concat--
100* Utility function. Concatenates together: dst, a space character and src. 100* Utility function. Concatenates together: dst, a space character and src.
101* dst + " " + src 101* dst + " " + src
102*/ 102*/
103void 103void
104concat(char **dst, const char *src) 104concat(char **dst, const char *src)
105{ 105{
106 concat2(dst, src, strlen(src)); 106 concat2(dst, src, strlen(src));
107} 107}
108 108
109void 109void
110concat2(char **dst, const char *src, size_t srclen) 110concat2(char **dst, const char *src, size_t srclen)
111{ 111{
112 size_t totallen, dstlen; 112 size_t totallen, dstlen;
113 assert(src != NULL); 113 assert(src != NULL);
114 114
115 /* 115 /*
116 * If destination buffer dst is NULL, then simply 116 * If destination buffer dst is NULL, then simply
117 * strdup the source buffer 117 * strdup the source buffer
118 */ 118 */
119 if (*dst == NULL) { 119 if (*dst == NULL) {
120 *dst = estrndup(src, srclen); 120 *dst = estrndup(src, srclen);
121 return; 121 return;
122 } 122 }
123 123
124 dstlen = strlen(*dst); 124 dstlen = strlen(*dst);
125 /* 125 /*
126 * NUL Byte and separator space 126 * NUL Byte and separator space
127 */ 127 */
128 totallen = dstlen + srclen + 2; 128 totallen = dstlen + srclen + 2;
129 129
130 *dst = erealloc(*dst, totallen); 130 *dst = erealloc(*dst, totallen);
131 131
132 /* Append a space at the end of dst */ 132 /* Append a space at the end of dst */
133 (*dst)[dstlen++] = ' '; 133 (*dst)[dstlen++] = ' ';
134 134
135 /* Now, copy src at the end of dst */ 135 /* Now, copy src at the end of dst */
136 memcpy(*dst + dstlen, src, srclen); 136 memcpy(*dst + dstlen, src, srclen);
137 (*dst)[dstlen + srclen] = '\0'; 137 (*dst)[dstlen + srclen] = '\0';
138} 138}
139 139
140void 140void
141close_db(sqlite3 *db) 141close_db(sqlite3 *db)
142{ 142{
143 sqlite3_close(db); 143 sqlite3_close(db);
144 sqlite3_shutdown(); 144 sqlite3_shutdown();
145} 145}
146 146
147/* 147/*
148 * create_db -- 148 * create_db --
149 * Creates the database schema. 149 * Creates the database schema.
150 */ 150 */
151static int 151static int
152create_db(sqlite3 *db) 152create_db(sqlite3 *db)
153{ 153{
154 const char *sqlstr = NULL; 154 const char *sqlstr = NULL;
155 char *schemasql; 155 char *schemasql;
156 char *errmsg = NULL; 156 char *errmsg = NULL;
157 157
158/*------------------------ Create the tables------------------------------*/ 158/*------------------------ Create the tables------------------------------*/
159 159
160#if NOTYET 160#if NOTYET
161 sqlite3_exec(db, "PRAGMA journal_mode = WAL", NULL, NULL, NULL); 161 sqlite3_exec(db, "PRAGMA journal_mode = WAL", NULL, NULL, NULL);
162#else 162#else
163 sqlite3_exec(db, "PRAGMA journal_mode = DELETE", NULL, NULL, NULL); 163 sqlite3_exec(db, "PRAGMA journal_mode = DELETE", NULL, NULL, NULL);
164#endif 164#endif
165 165
166 schemasql = sqlite3_mprintf("PRAGMA user_version = %d", 166 schemasql = sqlite3_mprintf("PRAGMA user_version = %d",
167 APROPOS_SCHEMA_VERSION); 167 APROPOS_SCHEMA_VERSION);
168 sqlite3_exec(db, schemasql, NULL, NULL, &errmsg); 168 sqlite3_exec(db, schemasql, NULL, NULL, &errmsg);
169 if (errmsg != NULL) 169 if (errmsg != NULL)
170 goto out; 170 goto out;
171 sqlite3_free(schemasql); 171 sqlite3_free(schemasql);
172 172
173 sqlstr = 173 sqlstr =
174 //mandb 174 //mandb
175 "CREATE VIRTUAL TABLE mandb USING fts4(section, name, " 175 "CREATE VIRTUAL TABLE mandb USING fts4(section, name, "
176 "name_desc, desc, lib, return_vals, env, files, " 176 "name_desc, desc, lib, return_vals, env, files, "
177 "exit_status, diagnostics, errors, md5_hash UNIQUE, machine, " 177 "exit_status, diagnostics, errors, md5_hash UNIQUE, machine, "
178 "compress=zip, uncompress=unzip, tokenize=porter); " 178 "compress=zip, uncompress=unzip, tokenize=porter, "
 179 "notindexed=section, notindexed=md5_hash); "
179 //mandb_meta 180 //mandb_meta
180 "CREATE TABLE IF NOT EXISTS mandb_meta(device, inode, mtime, " 181 "CREATE TABLE IF NOT EXISTS mandb_meta(device, inode, mtime, "
181 "file UNIQUE, md5_hash UNIQUE, id INTEGER PRIMARY KEY); " 182 "file UNIQUE, md5_hash UNIQUE, id INTEGER PRIMARY KEY); "
182 //mandb_links 183 //mandb_links
183 "CREATE TABLE IF NOT EXISTS mandb_links(link, target, section, " 184 "CREATE TABLE IF NOT EXISTS mandb_links(link, target, section, "
184 "machine, md5_hash); "; 185 "machine, md5_hash); ";
185 186
186 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 187 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
187 if (errmsg != NULL) 188 if (errmsg != NULL)
188 goto out; 189 goto out;
189 190
190 sqlstr = 191 sqlstr =
191 "CREATE INDEX IF NOT EXISTS index_mandb_links ON mandb_links " 192 "CREATE INDEX IF NOT EXISTS index_mandb_links ON mandb_links "
192 "(link); " 193 "(link); "
193 "CREATE INDEX IF NOT EXISTS index_mandb_meta_dev ON mandb_meta " 194 "CREATE INDEX IF NOT EXISTS index_mandb_meta_dev ON mandb_meta "
194 "(device, inode); " 195 "(device, inode); "
195 "CREATE INDEX IF NOT EXISTS index_mandb_links_md5 ON mandb_links " 196 "CREATE INDEX IF NOT EXISTS index_mandb_links_md5 ON mandb_links "
196 "(md5_hash);"; 197 "(md5_hash);";
197 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg); 198 sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
198 if (errmsg != NULL) 199 if (errmsg != NULL)
199 goto out; 200 goto out;
200 return 0; 201 return 0;
201 202
202out: 203out:
203 warnx("%s", errmsg); 204 warnx("%s", errmsg);
204 free(errmsg); 205 free(errmsg);
205 sqlite3_close(db); 206 sqlite3_close(db);
206 sqlite3_shutdown(); 207 sqlite3_shutdown();
207 return -1; 208 return -1;
208} 209}
209 210
210/* 211/*
211 * zip -- 212 * zip --
212 * User defined Sqlite function to compress the FTS table 213 * User defined Sqlite function to compress the FTS table
213 */ 214 */
214static void 215static void
215zip(sqlite3_context *pctx, int nval, sqlite3_value **apval) 216zip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
216{ 217{
217 int nin; 218 int nin;
218 long int nout; 219 long int nout;
219 const unsigned char * inbuf; 220 const unsigned char * inbuf;
220 unsigned char *outbuf; 221 unsigned char *outbuf;
221 222
222 assert(nval == 1); 223 assert(nval == 1);
223 nin = sqlite3_value_bytes(apval[0]); 224 nin = sqlite3_value_bytes(apval[0]);
224 inbuf = (const unsigned char *) sqlite3_value_blob(apval[0]); 225 inbuf = (const unsigned char *) sqlite3_value_blob(apval[0]);
225 nout = nin + 13 + (nin + 999) / 1000; 226 nout = nin + 13 + (nin + 999) / 1000;
226 outbuf = emalloc(nout); 227 outbuf = emalloc(nout);
227 compress(outbuf, (unsigned long *) &nout, inbuf, nin); 228 compress(outbuf, (unsigned long *) &nout, inbuf, nin);
228 sqlite3_result_blob(pctx, outbuf, nout, free); 229 sqlite3_result_blob(pctx, outbuf, nout, free);
229} 230}
230 231
231/* 232/*
232 * unzip -- 233 * unzip --
233 * User defined Sqlite function to uncompress the FTS table. 234 * User defined Sqlite function to uncompress the FTS table.
234 */ 235 */
235static void 236static void
236unzip(sqlite3_context *pctx, int nval, sqlite3_value **apval) 237unzip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
237{ 238{
238 unsigned int rc; 239 unsigned int rc;
239 unsigned char *outbuf; 240 unsigned char *outbuf;
240 z_stream stream; 241 z_stream stream;
241 242
242 assert(nval == 1); 243 assert(nval == 1);
243 stream.next_in = __UNCONST(sqlite3_value_blob(apval[0])); 244 stream.next_in = __UNCONST(sqlite3_value_blob(apval[0]));
244 stream.avail_in = sqlite3_value_bytes(apval[0]); 245 stream.avail_in = sqlite3_value_bytes(apval[0]);
245 stream.avail_out = stream.avail_in * 2 + 100; 246 stream.avail_out = stream.avail_in * 2 + 100;
246 stream.next_out = outbuf = emalloc(stream.avail_out); 247 stream.next_out = outbuf = emalloc(stream.avail_out);
247 stream.zalloc = NULL; 248 stream.zalloc = NULL;
248 stream.zfree = NULL; 249 stream.zfree = NULL;
249 250
250 if (inflateInit(&stream) != Z_OK) { 251 if (inflateInit(&stream) != Z_OK) {
251 free(outbuf); 252 free(outbuf);
252 return; 253 return;
253 } 254 }
254 255
255 while ((rc = inflate(&stream, Z_SYNC_FLUSH)) != Z_STREAM_END) { 256 while ((rc = inflate(&stream, Z_SYNC_FLUSH)) != Z_STREAM_END) {
256 if (rc != Z_OK || 257 if (rc != Z_OK ||
257 (stream.avail_out != 0 && stream.avail_in == 0)) { 258 (stream.avail_out != 0 && stream.avail_in == 0)) {
258 free(outbuf); 259 free(outbuf);
259 return; 260 return;
260 } 261 }
261 outbuf = erealloc(outbuf, stream.total_out * 2); 262 outbuf = erealloc(outbuf, stream.total_out * 2);
262 stream.next_out = outbuf + stream.total_out; 263 stream.next_out = outbuf + stream.total_out;
263 stream.avail_out = stream.total_out; 264 stream.avail_out = stream.total_out;
264 } 265 }
265 if (inflateEnd(&stream) != Z_OK) { 266 if (inflateEnd(&stream) != Z_OK) {
266 free(outbuf); 267 free(outbuf);
267 return; 268 return;
268 } 269 }
269 outbuf = erealloc(outbuf, stream.total_out); 270 outbuf = erealloc(outbuf, stream.total_out);
270 sqlite3_result_text(pctx, (const char *)outbuf, stream.total_out, free); 271 sqlite3_result_text(pctx, (const char *)outbuf, stream.total_out, free);
271} 272}
272 273
273/* 274/*
274 * get_dbpath -- 275 * get_dbpath --
275 * Read the path of the database from man.conf and return. 276 * Read the path of the database from man.conf and return.
276 */ 277 */
277char * 278char *
278get_dbpath(const char *manconf) 279get_dbpath(const char *manconf)
279{ 280{
280 TAG *tp; 281 TAG *tp;
281 char *dbpath; 282 char *dbpath;
282 283
283 config(manconf); 284 config(manconf);
284 tp = gettag("_mandb", 1); 285 tp = gettag("_mandb", 1);
285 if (!tp) 286 if (!tp)
286 return NULL; 287 return NULL;
287 288
288 if (TAILQ_EMPTY(&tp->entrylist)) 289 if (TAILQ_EMPTY(&tp->entrylist))
289 return NULL; 290 return NULL;
290 291
291 dbpath = TAILQ_LAST(&tp->entrylist, tqh)->s; 292 dbpath = TAILQ_LAST(&tp->entrylist, tqh)->s;
292 return dbpath; 293 return dbpath;
293} 294}
294 295
295/* init_db -- 296/* init_db --
296 * Prepare the database. Register the compress/uncompress functions and the 297 * Prepare the database. Register the compress/uncompress functions and the
297 * stopword tokenizer. 298 * stopword tokenizer.
298 * db_flag specifies the mode in which to open the database. 3 options are 299 * db_flag specifies the mode in which to open the database. 3 options are
299 * available: 300 * available:
300 * 1. DB_READONLY: Open in READONLY mode. An error if db does not exist. 301 * 1. DB_READONLY: Open in READONLY mode. An error if db does not exist.
301 * 2. DB_READWRITE: Open in read-write mode. An error if db does not exist. 302 * 2. DB_READWRITE: Open in read-write mode. An error if db does not exist.
302 * 3. DB_CREATE: Open in read-write mode. It will try to create the db if 303 * 3. DB_CREATE: Open in read-write mode. It will try to create the db if
303 * it does not exist already. 304 * it does not exist already.
304 * RETURN VALUES: 305 * RETURN VALUES:
305 * The function will return NULL in case the db does not exist 306 * The function will return NULL in case the db does not exist
306 * and DB_CREATE 307 * and DB_CREATE
307 * was not specified. And in case DB_CREATE was specified and yet NULL is 308 * was not specified. And in case DB_CREATE was specified and yet NULL is
308 * returned, then there was some other error. 309 * returned, then there was some other error.
309 * In normal cases the function should return a handle to the db. 310 * In normal cases the function should return a handle to the db.
310 */ 311 */
311sqlite3 * 312sqlite3 *
312init_db(mandb_access_mode db_flag, const char *manconf) 313init_db(mandb_access_mode db_flag, const char *manconf)
313{ 314{
314 sqlite3 *db = NULL; 315 sqlite3 *db = NULL;
315 sqlite3_stmt *stmt; 316 sqlite3_stmt *stmt;
316 struct stat sb; 317 struct stat sb;
317 int rc; 318 int rc;
318 int create_db_flag = 0; 319 int create_db_flag = 0;
319 320
320 char *dbpath = get_dbpath(manconf); 321 char *dbpath = get_dbpath(manconf);
321 if (dbpath == NULL) 322 if (dbpath == NULL)
322 errx(EXIT_FAILURE, "_mandb entry not found in man.conf"); 323 errx(EXIT_FAILURE, "_mandb entry not found in man.conf");
323 324
324 if (!(stat(dbpath, &sb) == 0 && S_ISREG(sb.st_mode))) { 325 if (!(stat(dbpath, &sb) == 0 && S_ISREG(sb.st_mode))) {
325 /* Database does not exist, check if DB_CREATE was specified, 326 /* Database does not exist, check if DB_CREATE was specified,
326 * and set flag to create the database schema 327 * and set flag to create the database schema
327 */ 328 */
328 if (db_flag != (MANDB_CREATE)) { 329 if (db_flag != (MANDB_CREATE)) {
329 warnx("Missing apropos database. " 330 warnx("Missing apropos database. "
330 "Please run makemandb to create it."); 331 "Please run makemandb to create it.");
331 return NULL; 332 return NULL;
332 } 333 }
333 create_db_flag = 1; 334 create_db_flag = 1;
334 } else { 335 } else {
335 /* 336 /*
336 * Database exists. Check if we have the permissions 337 * Database exists. Check if we have the permissions
337 * to read/write the files 338 * to read/write the files
338 */ 339 */
339 int access_mode = R_OK; 340 int access_mode = R_OK;
340 switch (db_flag) { 341 switch (db_flag) {
341 case MANDB_CREATE: 342 case MANDB_CREATE:
342 case MANDB_WRITE: 343 case MANDB_WRITE:
343 access_mode |= W_OK; 344 access_mode |= W_OK;
344 break; 345 break;
345 default: 346 default:
346 break; 347 break;
347 } 348 }
348 if ((access(dbpath, access_mode)) != 0) { 349 if ((access(dbpath, access_mode)) != 0) {
349 warnx("Unable to access the database, please check" 350 warnx("Unable to access the database, please check"
350 " permissions for `%s'", dbpath); 351 " permissions for `%s'", dbpath);
351 return NULL; 352 return NULL;
352 } 353 }
353 } 354 }
354 355
355 sqlite3_initialize(); 356 sqlite3_initialize();
356 rc = sqlite3_open_v2(dbpath, &db, db_flag, NULL); 357 rc = sqlite3_open_v2(dbpath, &db, db_flag, NULL);
357 358
358 if (rc != SQLITE_OK) { 359 if (rc != SQLITE_OK) {
359 warnx("%s", sqlite3_errmsg(db)); 360 warnx("%s", sqlite3_errmsg(db));
360 goto error; 361 goto error;
361 } 362 }
362 363
363 if (create_db_flag && create_db(db) < 0) { 364 if (create_db_flag && create_db(db) < 0) {
364 warnx("%s", "Unable to create database schema"); 365 warnx("%s", "Unable to create database schema");
365 goto error; 366 goto error;
366 } 367 }
367 368
368 rc = sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &stmt, NULL); 369 rc = sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &stmt, NULL);
369 if (rc != SQLITE_OK) { 370 if (rc != SQLITE_OK) {
370 warnx("Unable to query schema version: %s", 371 warnx("Unable to query schema version: %s",
371 sqlite3_errmsg(db)); 372 sqlite3_errmsg(db));
372 goto error; 373 goto error;
373 } 374 }
374 if (sqlite3_step(stmt) != SQLITE_ROW) { 375 if (sqlite3_step(stmt) != SQLITE_ROW) {
375 sqlite3_finalize(stmt); 376 sqlite3_finalize(stmt);
376 warnx("Unable to query schema version: %s", 377 warnx("Unable to query schema version: %s",
377 sqlite3_errmsg(db)); 378 sqlite3_errmsg(db));
378 goto error; 379 goto error;
379 } 380 }
380 if (sqlite3_column_int(stmt, 0) != APROPOS_SCHEMA_VERSION) { 381 if (sqlite3_column_int(stmt, 0) != APROPOS_SCHEMA_VERSION) {
381 sqlite3_finalize(stmt); 382 sqlite3_finalize(stmt);
382 warnx("Incorrect schema version found. " 383 warnx("Incorrect schema version found. "
383 "Please run makemandb -f."); 384 "Please run makemandb -f.");
384 goto error; 385 goto error;
385 } 386 }
386 sqlite3_finalize(stmt); 387 sqlite3_finalize(stmt);
387 388
388 sqlite3_extended_result_codes(db, 1); 389 sqlite3_extended_result_codes(db, 1);
389 390
390 /* Register the zip and unzip functions for FTS compression */ 391 /* Register the zip and unzip functions for FTS compression */
391 rc = sqlite3_create_function(db, "zip", 1, SQLITE_ANY, NULL, zip, 392 rc = sqlite3_create_function(db, "zip", 1, SQLITE_ANY, NULL, zip,
392 NULL, NULL); 393 NULL, NULL);
393 if (rc != SQLITE_OK) { 394 if (rc != SQLITE_OK) {
394 warnx("Unable to register function: compress: %s", 395 warnx("Unable to register function: compress: %s",
395 sqlite3_errmsg(db)); 396 sqlite3_errmsg(db));
396 goto error; 397 goto error;
397 } 398 }
398 399
399 rc = sqlite3_create_function(db, "unzip", 1, SQLITE_ANY, NULL, 400 rc = sqlite3_create_function(db, "unzip", 1, SQLITE_ANY, NULL,
400 unzip, NULL, NULL); 401 unzip, NULL, NULL);
401 if (rc != SQLITE_OK) { 402 if (rc != SQLITE_OK) {
402 warnx("Unable to register function: uncompress: %s", 403 warnx("Unable to register function: uncompress: %s",
403 sqlite3_errmsg(db)); 404 sqlite3_errmsg(db));
404 goto error; 405 goto error;
405 } 406 }
406 return db; 407 return db;
407 408
408error: 409error:
409 close_db(db); 410 close_db(db);
410 return NULL; 411 return NULL;
411} 412}
412 413
413/* 414/*
414 * rank_func -- 415 * rank_func --
415 * Sqlite user defined function for ranking the documents. 416 * Sqlite user defined function for ranking the documents.
416 * For each phrase of the query, it computes the tf and idf and adds them over. 417 * For each phrase of the query, it computes the tf and idf and adds them over.
417 * It computes the final rank, by multiplying tf and idf together. 418 * It computes the final rank, by multiplying tf and idf together.
418 * Weight of term t for document d = (term frequency of t in d * 419 * Weight of term t for document d = (term frequency of t in d *
419 * inverse document frequency of t) 420 * inverse document frequency of t)
420 * 421 *
421 * Term Frequency of term t in document d = Number of times t occurs in d / 422 * Term Frequency of term t in document d = Number of times t occurs in d /
422 * Number of times t appears in all documents 423 * Number of times t appears in all documents
423 * 424 *
424 * Inverse document frequency of t = log(Total number of documents / 425 * Inverse document frequency of t = log(Total number of documents /
425 * Number of documents in which t occurs) 426 * Number of documents in which t occurs)
426 */ 427 */
427static void 428static void
428rank_func(sqlite3_context *pctx, int nval, sqlite3_value **apval) 429rank_func(sqlite3_context *pctx, int nval, sqlite3_value **apval)
429{ 430{
430 inverse_document_frequency *idf = sqlite3_user_data(pctx); 431 inverse_document_frequency *idf = sqlite3_user_data(pctx);
431 double tf = 0.0; 432 double tf = 0.0;
432 const unsigned int *matchinfo; 433 const unsigned int *matchinfo;
433 int ncol; 434 int ncol;
434 int nphrase; 435 int nphrase;
435 int iphrase; 436 int iphrase;
436 int ndoc; 437 int ndoc;
437 int doclen = 0; 438 int doclen = 0;
438 const double k = 3.75; 439 const double k = 3.75;
439 /* 440 /*
440 * Check that the number of arguments passed to this 441 * Check that the number of arguments passed to this
441 * function is correct. 442 * function is correct.
442 */ 443 */
443 assert(nval == 1); 444 assert(nval == 1);
444 445
445 matchinfo = (const unsigned int *) sqlite3_value_blob(apval[0]); 446 matchinfo = (const unsigned int *) sqlite3_value_blob(apval[0]);
446 nphrase = matchinfo[0]; 447 nphrase = matchinfo[0];
447 ncol = matchinfo[1]; 448 ncol = matchinfo[1];
448 ndoc = matchinfo[2 + 3 * ncol * nphrase + ncol]; 449 ndoc = matchinfo[2 + 3 * ncol * nphrase + ncol];
449 for (iphrase = 0; iphrase < nphrase; iphrase++) { 450 for (iphrase = 0; iphrase < nphrase; iphrase++) {
450 int icol; 451 int icol;
451 const unsigned int *phraseinfo = 452 const unsigned int *phraseinfo =
452 &matchinfo[2 + ncol + iphrase * ncol * 3]; 453 &matchinfo[2 + ncol + iphrase * ncol * 3];
453 for(icol = 1; icol < ncol; icol++) { 454 for(icol = 1; icol < ncol; icol++) {
454 455
455 /* nhitcount: number of times the current phrase occurs 456 /* nhitcount: number of times the current phrase occurs
456 * in the current column in the current document. 457 * in the current column in the current document.
457 * nglobalhitcount: number of times current phrase 458 * nglobalhitcount: number of times current phrase
458 * occurs in the current column in all documents. 459 * occurs in the current column in all documents.
459 * ndocshitcount: number of documents in which the 460 * ndocshitcount: number of documents in which the
460 * current phrase occurs in the current column at 461 * current phrase occurs in the current column at
461 * least once. 462 * least once.
462 */ 463 */
463 int nhitcount = phraseinfo[3 * icol]; 464 int nhitcount = phraseinfo[3 * icol];
464 int nglobalhitcount = phraseinfo[3 * icol + 1]; 465 int nglobalhitcount = phraseinfo[3 * icol + 1];
465 int ndocshitcount = phraseinfo[3 * icol + 2]; 466 int ndocshitcount = phraseinfo[3 * icol + 2];
466 doclen = matchinfo[2 + icol ]; 467 doclen = matchinfo[2 + icol ];
467 double weight = col_weights[icol - 1]; 468 double weight = col_weights[icol - 1];
468 if (idf->status == 0 && ndocshitcount) 469 if (idf->status == 0 && ndocshitcount)
469 idf->value += 470 idf->value +=
470 log(((double)ndoc / ndocshitcount))* weight; 471 log(((double)ndoc / ndocshitcount))* weight;
471 472
472 /* 473 /*
473 * Dividing the tf by document length to normalize 474 * Dividing the tf by document length to normalize
474 * the effect of longer documents. 475 * the effect of longer documents.
475 */ 476 */
476 if (nglobalhitcount > 0 && nhitcount) 477 if (nglobalhitcount > 0 && nhitcount)
477 tf += (((double)nhitcount * weight) 478 tf += (((double)nhitcount * weight)
478 / (nglobalhitcount * doclen)); 479 / (nglobalhitcount * doclen));
479 } 480 }
480 } 481 }
481 idf->status = 1; 482 idf->status = 1;
482 483
483 /* 484 /*
484 * Final score: Dividing by k + tf further normalizes the weight 485 * Final score: Dividing by k + tf further normalizes the weight
485 * leading to better results. The value of k is experimental 486 * leading to better results. The value of k is experimental
486 */ 487 */
487 double score = (tf * idf->value) / (k + tf); 488 double score = (tf * idf->value) / (k + tf);
488 sqlite3_result_double(pctx, score); 489 sqlite3_result_double(pctx, score);
489 return; 490 return;
490} 491}
491 492
492/* 493/*
493 * generates sql query for matching the user entered query 494 * generates sql query for matching the user entered query
494 */ 495 */
495static char * 496static char *
496generate_search_query(query_args *args, const char *snippet_args[3]) 497generate_search_query(query_args *args, const char *snippet_args[3])
497{ 498{
498 const char *default_snippet_args[3]; 499 const char *default_snippet_args[3];
499 char *section_clause = NULL; 500 char *section_clause = NULL;
500 char *limit_clause = NULL; 501 char *limit_clause = NULL;
501 char *machine_clause = NULL; 502 char *machine_clause = NULL;
502 char *query; 503 char *query;
503 504
504 if (args->machine) 505 if (args->machine)
505 easprintf(&machine_clause, "AND machine = \'%s\' ", 506 easprintf(&machine_clause, "AND machine = \'%s\' ",
506 args->machine); 507 args->machine);
507 508
508 509
509 /* We want to build a query of the form: "select x,y,z from mandb where 510 /* We want to build a query of the form: "select x,y,z from mandb where
510 * mandb match :query [AND (section LIKE '1' OR section LIKE '2' OR...)] 511 * mandb match :query [AND (section LIKE '1' OR section LIKE '2' OR...)]
511 * ORDER BY rank DESC..." 512 * ORDER BY rank DESC..."
512 * NOTES: 513 * NOTES:
513 * 1. The portion in square brackets is optional, it will be there 514 * 1. The portion in square brackets is optional, it will be there
514 * only if the user has specified an option on the command line 515 * only if the user has specified an option on the command line
515 * to search in one or more specific sections. 516 * to search in one or more specific sections.
516 */ 517 */
517 char *sections_str = args->sec_nums; 518 char *sections_str = args->sec_nums;
518 char *temp; 519 char *temp;
519 if (sections_str) { 520 if (sections_str) {
520 while (*sections_str) { 521 while (*sections_str) {
521 size_t len = strcspn(sections_str, " "); 522 size_t len = strcspn(sections_str, " ");
522 char *sec = sections_str; 523 char *sec = sections_str;
523 if (sections_str[len] == 0) { 524 if (sections_str[len] == 0) {
524 sections_str += len; 525 sections_str += len;
525 } else { 526 } else {
526 sections_str[len] = 0; 527 sections_str[len] = 0;
527 sections_str += len + 1; 528 sections_str += len + 1;
528 } 529 }
529 easprintf(&temp, "\'%s\',", sec); 530 easprintf(&temp, "\'%s\',", sec);
530 531
531 if (section_clause) { 532 if (section_clause) {
532 concat(&section_clause, temp); 533 concat(&section_clause, temp);
533 free(temp); 534 free(temp);
534 } else { 535 } else {
535 section_clause = temp; 536 section_clause = temp;
536 } 537 }
537 } 538 }
538 if (section_clause) { 539 if (section_clause) {
539 /* 540 /*
540 * At least one section requested, add glue for query. 541 * At least one section requested, add glue for query.
541 * Before doing that, remove the comma at the end of 542 * Before doing that, remove the comma at the end of
542 * section_clause 543 * section_clause
543 */ 544 */
544 size_t section_clause_len = strlen(section_clause); 545 size_t section_clause_len = strlen(section_clause);
545 if (section_clause[section_clause_len - 1] == ',') 546 if (section_clause[section_clause_len - 1] == ',')
546 section_clause[section_clause_len - 1] = 0; 547 section_clause[section_clause_len - 1] = 0;
547 temp = section_clause; 548 temp = section_clause;
548 easprintf(&section_clause, " AND section IN (%s)", temp); 549 easprintf(&section_clause, " AND section IN (%s)", temp);
549 free(temp); 550 free(temp);
550 } 551 }
551 } 552 }
552 553
553 if (args->nrec >= 0) { 554 if (args->nrec >= 0) {
554 /* Use the provided number of records and offset */ 555 /* Use the provided number of records and offset */
555 easprintf(&limit_clause, " LIMIT %d OFFSET %d", 556 easprintf(&limit_clause, " LIMIT %d OFFSET %d",
556 args->nrec, args->offset); 557 args->nrec, args->offset);
557 } 558 }
558 559
559 if (snippet_args == NULL) { 560 if (snippet_args == NULL) {
560 default_snippet_args[0] = ""; 561 default_snippet_args[0] = "";
561 default_snippet_args[1] = ""; 562 default_snippet_args[1] = "";
562 default_snippet_args[2] = "..."; 563 default_snippet_args[2] = "...";
563 snippet_args = default_snippet_args; 564 snippet_args = default_snippet_args;
564 } 565 }
565 566
566 if (args->legacy) { 567 if (args->legacy) {
567 char *wild; 568 char *wild;
568 easprintf(&wild, "%%%s%%", args->search_str); 569 easprintf(&wild, "%%%s%%", args->search_str);
569 query = sqlite3_mprintf("SELECT section, name, name_desc, machine" 570 query = sqlite3_mprintf("SELECT section, name, name_desc, machine"
570 " FROM mandb" 571 " FROM mandb"
571 " WHERE name LIKE %Q OR name_desc LIKE %Q " 572 " WHERE name LIKE %Q OR name_desc LIKE %Q "
572 "%s" 573 "%s"
573 "%s", 574 "%s",
574 wild, wild, 575 wild, wild,
575 section_clause ? section_clause : "", 576 section_clause ? section_clause : "",
576 limit_clause ? limit_clause : ""); 577 limit_clause ? limit_clause : "");
577 free(wild); 578 free(wild);
578 } else { 579 } else {
579 query = sqlite3_mprintf("SELECT section, name, name_desc, machine," 580 query = sqlite3_mprintf("SELECT section, name, name_desc, machine,"
580 " snippet(mandb, %Q, %Q, %Q, -1, 40 )," 581 " snippet(mandb, %Q, %Q, %Q, -1, 40 ),"
581 " rank_func(matchinfo(mandb, \"pclxn\")) AS rank" 582 " rank_func(matchinfo(mandb, \"pclxn\")) AS rank"
582 " FROM mandb" 583 " FROM mandb"
583 " WHERE mandb MATCH %Q %s " 584 " WHERE mandb MATCH %Q %s "
584 "%s" 585 "%s"
585 " ORDER BY rank DESC" 586 " ORDER BY rank DESC"
586 "%s", 587 "%s",
587 snippet_args[0], snippet_args[1], snippet_args[2], 588 snippet_args[0], snippet_args[1], snippet_args[2],
588 args->search_str, machine_clause ? machine_clause : "", 589 args->search_str, machine_clause ? machine_clause : "",
589 section_clause ? section_clause : "", 590 section_clause ? section_clause : "",
590 limit_clause ? limit_clause : ""); 591 limit_clause ? limit_clause : "");
591 } 592 }
592 593
593 free(machine_clause); 594 free(machine_clause);
594 free(section_clause); 595 free(section_clause);
595 free(limit_clause); 596 free(limit_clause);
596 return query; 597 return query;
597} 598}
598 599
599/* 600/*
600 * Execute the full text search query and return the number of results 601 * Execute the full text search query and return the number of results
601 * obtained. 602 * obtained.
602 */ 603 */
603static unsigned int 604static unsigned int
604execute_search_query(sqlite3 *db, char *query, query_args *args) 605execute_search_query(sqlite3 *db, char *query, query_args *args)
605{ 606{
606 sqlite3_stmt *stmt; 607 sqlite3_stmt *stmt;
607 const char *section; 608 const char *section;
608 char *name; 609 char *name;
609 char *slash_ptr; 610 char *slash_ptr;
610 const char *name_desc; 611 const char *name_desc;
611 const char *machine; 612 const char *machine;
612 const char *snippet = ""; 613 const char *snippet = "";
613 const char *name_temp; 614 const char *name_temp;
614 char *m = NULL; 615 char *m = NULL;
615 int rc; 616 int rc;
616 inverse_document_frequency idf = {0, 0}; 617 inverse_document_frequency idf = {0, 0};
617 618
618 if (!args->legacy) { 619 if (!args->legacy) {
619 /* Register the rank function */ 620 /* Register the rank function */
620 rc = sqlite3_create_function(db, "rank_func", 1, SQLITE_ANY, 621 rc = sqlite3_create_function(db, "rank_func", 1, SQLITE_ANY,
621 (void *) &idf, rank_func, NULL, NULL); 622 (void *) &idf, rank_func, NULL, NULL);
622 if (rc != SQLITE_OK) { 623 if (rc != SQLITE_OK) {
623 warnx("Unable to register the ranking function: %s", 624 warnx("Unable to register the ranking function: %s",
624 sqlite3_errmsg(db)); 625 sqlite3_errmsg(db));
625 sqlite3_close(db); 626 sqlite3_close(db);
626 sqlite3_shutdown(); 627 sqlite3_shutdown();
627 exit(EXIT_FAILURE); 628 exit(EXIT_FAILURE);
628 } 629 }
629 } 630 }
630 631
631 rc = sqlite3_prepare_v2(db, query, -1, &stmt, NULL); 632 rc = sqlite3_prepare_v2(db, query, -1, &stmt, NULL);
632 if (rc == SQLITE_IOERR) { 633 if (rc == SQLITE_IOERR) {
633 warnx("Corrupt database. Please rerun makemandb"); 634 warnx("Corrupt database. Please rerun makemandb");
634 return -1; 635 return -1;
635 } else if (rc != SQLITE_OK) { 636 } else if (rc != SQLITE_OK) {
636 warnx("%s", sqlite3_errmsg(db)); 637 warnx("%s", sqlite3_errmsg(db));
637 return -1; 638 return -1;
638 } 639 }
639 640
640 unsigned int nresults = 0; 641 unsigned int nresults = 0;
641 while (sqlite3_step(stmt) == SQLITE_ROW) { 642 while (sqlite3_step(stmt) == SQLITE_ROW) {
642 nresults++; 643 nresults++;
643 section = (const char *) sqlite3_column_text(stmt, 0); 644 section = (const char *) sqlite3_column_text(stmt, 0);
644 name_temp = (const char *) sqlite3_column_text(stmt, 1); 645 name_temp = (const char *) sqlite3_column_text(stmt, 1);
645 name_desc = (const char *) sqlite3_column_text(stmt, 2); 646 name_desc = (const char *) sqlite3_column_text(stmt, 2);
646 machine = (const char *) sqlite3_column_text(stmt, 3); 647 machine = (const char *) sqlite3_column_text(stmt, 3);
647 if (!args->legacy) 648 if (!args->legacy)
648 snippet = (const char *) sqlite3_column_text(stmt, 4); 649 snippet = (const char *) sqlite3_column_text(stmt, 4);
649 if ((slash_ptr = strrchr(name_temp, '/')) != NULL) 650 if ((slash_ptr = strrchr(name_temp, '/')) != NULL)
650 name_temp = slash_ptr + 1; 651 name_temp = slash_ptr + 1;
651 if (machine && machine[0]) { 652 if (machine && machine[0]) {
652 m = estrdup(machine); 653 m = estrdup(machine);
653 easprintf(&name, "%s/%s", lower(m), name_temp); 654 easprintf(&name, "%s/%s", lower(m), name_temp);
654 free(m); 655 free(m);
655 } else { 656 } else {
656 name = estrdup((const char *) 657 name = estrdup((const char *)
657 sqlite3_column_text(stmt, 1)); 658 sqlite3_column_text(stmt, 1));
658 } 659 }
659 660
660 (args->callback)(args->callback_data, section, name, 661 (args->callback)(args->callback_data, section, name,
661 name_desc, snippet, args->legacy? 0: strlen(snippet)); 662 name_desc, snippet, args->legacy? 0: strlen(snippet));
662 free(name); 663 free(name);
663 } 664 }
664 sqlite3_finalize(stmt); 665 sqlite3_finalize(stmt);
665 return nresults; 666 return nresults;
666} 667}
667 668
668 669
669/* 670/*
670 * run_query_internal -- 671 * run_query_internal --
671 * Performs the searches for the keywords entered by the user. 672 * Performs the searches for the keywords entered by the user.
672 * The 2nd param: snippet_args is an array of strings providing values for the 673 * The 2nd param: snippet_args is an array of strings providing values for the
673 * last three parameters to the snippet function of sqlite. (Look at the docs). 674 * last three parameters to the snippet function of sqlite. (Look at the docs).
674 * The 3rd param: args contains rest of the search parameters. Look at 675 * The 3rd param: args contains rest of the search parameters. Look at
675 * arpopos-utils.h for the description of individual fields. 676 * arpopos-utils.h for the description of individual fields.
676 * 677 *
677 */ 678 */
678static int 679static int
679run_query_internal(sqlite3 *db, const char *snippet_args[3], query_args *args) 680run_query_internal(sqlite3 *db, const char *snippet_args[3], query_args *args)
680{ 681{
681 char *query; 682 char *query;
682 query = generate_search_query(args, snippet_args); 683 query = generate_search_query(args, snippet_args);
683 if (query == NULL) { 684 if (query == NULL) {
684 *args->errmsg = estrdup("malloc failed"); 685 *args->errmsg = estrdup("malloc failed");
685 return -1; 686 return -1;
686 } 687 }
687 688
688 execute_search_query(db, query, args); 689 execute_search_query(db, query, args);
689 sqlite3_free(query); 690 sqlite3_free(query);
690 return *(args->errmsg) == NULL ? 0 : -1; 691 return *(args->errmsg) == NULL ? 0 : -1;
691} 692}
692 693
693static char * 694static char *
694get_escaped_html_string(const char *src, size_t *slen) 695get_escaped_html_string(const char *src, size_t *slen)
695{ 696{
696 static const char trouble[] = "<>\"&\002\003"; 697 static const char trouble[] = "<>\"&\002\003";
697 /* 698 /*
698 * First scan the src to find out the number of occurrences 699 * First scan the src to find out the number of occurrences
699 * of {'>', '<' '"', '&'}. Then allocate a new buffer with 700 * of {'>', '<' '"', '&'}. Then allocate a new buffer with
700 * sufficient space to be able to store the quoted versions 701 * sufficient space to be able to store the quoted versions
701 * of the special characters {&gt;, &lt;, &quot;, &amp;}. 702 * of the special characters {&gt;, &lt;, &quot;, &amp;}.
702 * Copy over the characters from the original src into 703 * Copy over the characters from the original src into
703 * this buffer while replacing the special characters with 704 * this buffer while replacing the special characters with
704 * their quoted versions. 705 * their quoted versions.
705 */ 706 */
706 char *dst, *ddst; 707 char *dst, *ddst;
707 size_t count; 708 size_t count;
708 const char *ssrc; 709 const char *ssrc;
709 710
710 for (count = 0, ssrc = src; *src; count++) { 711 for (count = 0, ssrc = src; *src; count++) {
711 size_t sz = strcspn(src, trouble); 712 size_t sz = strcspn(src, trouble);
712 src += sz + 1; 713 src += sz + 1;
713 } 714 }
714 715
715 716
716#define append(a) \ 717#define append(a) \
717 do { \ 718 do { \
718 memcpy(dst, (a), sizeof(a) - 1); \ 719 memcpy(dst, (a), sizeof(a) - 1); \
719 dst += sizeof(a) - 1; \ 720 dst += sizeof(a) - 1; \
720 } while (/*CONSTCOND*/0) 721 } while (/*CONSTCOND*/0)
721 722
722 723
723 ddst = dst = emalloc(*slen + count * 5 + 1); 724 ddst = dst = emalloc(*slen + count * 5 + 1);
724 for (src = ssrc; *src; src++) { 725 for (src = ssrc; *src; src++) {
725 switch (*src) { 726 switch (*src) {
726 case '<': 727 case '<':
727 append("&lt;"); 728 append("&lt;");
728 break; 729 break;
729 case '>': 730 case '>':
730 append("&gt;"); 731 append("&gt;");
731 break; 732 break;
732 case '\"': 733 case '\"':
733 append("&quot;"); 734 append("&quot;");
734 break; 735 break;
735 case '&': 736 case '&':
736 /* 737 /*
737 * Don't perform the quoting if this & is part of 738 * Don't perform the quoting if this & is part of
738 * an mdoc escape sequence, e.g. \& 739 * an mdoc escape sequence, e.g. \&
739 */ 740 */
740 if (src != ssrc && src[-1] != '\\') 741 if (src != ssrc && src[-1] != '\\')
741 append("&amp;"); 742 append("&amp;");
742 else 743 else
743 append("&"); 744 append("&");
744 break; 745 break;
745 case '\002': 746 case '\002':
746 append("<b>"); 747 append("<b>");
747 break; 748 break;
748 case '\003': 749 case '\003':
749 append("</b>"); 750 append("</b>");
750 break; 751 break;
751 default: 752 default:
752 *dst++ = *src; 753 *dst++ = *src;
753 break; 754 break;
754 } 755 }
755 } 756 }
756 *dst = '\0'; 757 *dst = '\0';
757 *slen = dst - ddst; 758 *slen = dst - ddst;
758 return ddst; 759 return ddst;
759} 760}
760 761
761 762
762/* 763/*
763 * callback_html -- 764 * callback_html --
764 * Callback function for run_query_html. It builds the html output and then 765 * Callback function for run_query_html. It builds the html output and then
765 * calls the actual user supplied callback function. 766 * calls the actual user supplied callback function.
766 */ 767 */
767static int 768static int
768callback_html(void *data, const char *section, const char *name, 769callback_html(void *data, const char *section, const char *name,
769 const char *name_desc, const char *snippet, size_t snippet_length) 770 const char *name_desc, const char *snippet, size_t snippet_length)
770{ 771{
771 struct orig_callback_data *orig_data = data; 772 struct orig_callback_data *orig_data = data;
772 int (*callback)(void *, const char *, const char *, const char *, 773 int (*callback)(void *, const char *, const char *, const char *,
773 const char *, size_t) = orig_data->callback; 774 const char *, size_t) = orig_data->callback;
774 size_t length = snippet_length; 775 size_t length = snippet_length;
775 size_t name_description_length = strlen(name_desc); 776 size_t name_description_length = strlen(name_desc);
776 char *qsnippet = get_escaped_html_string(snippet, &length); 777 char *qsnippet = get_escaped_html_string(snippet, &length);
777 char *qname_description = get_escaped_html_string(name_desc, 778 char *qname_description = get_escaped_html_string(name_desc,
778 &name_description_length); 779 &name_description_length);
779 780
780 (*callback)(orig_data->data, section, name, qname_description, 781 (*callback)(orig_data->data, section, name, qname_description,
781 qsnippet, length); 782 qsnippet, length);
782 free(qsnippet); 783 free(qsnippet);
783 free(qname_description); 784 free(qname_description);
784 return 0; 785 return 0;
785} 786}
786 787
787/* 788/*
788 * run_query_html -- 789 * run_query_html --
789 * Utility function to output query result in HTML format. 790 * Utility function to output query result in HTML format.
790 * It internally calls run_query only, but it first passes the output to its 791 * It internally calls run_query only, but it first passes the output to its
791 * own custom callback function, which preprocess the snippet for quoting 792 * own custom callback function, which preprocess the snippet for quoting
792 * inline HTML fragments. 793 * inline HTML fragments.
793 * After that it delegates the call the actual user supplied callback function. 794 * After that it delegates the call the actual user supplied callback function.
794 */ 795 */
795static int 796static int
796run_query_html(sqlite3 *db, query_args *args) 797run_query_html(sqlite3 *db, query_args *args)
797{ 798{
798 struct orig_callback_data orig_data; 799 struct orig_callback_data orig_data;
799 orig_data.callback = args->callback; 800 orig_data.callback = args->callback;
800 orig_data.data = args->callback_data; 801 orig_data.data = args->callback_data;
801 const char *snippet_args[] = {"\002", "\003", "..."}; 802 const char *snippet_args[] = {"\002", "\003", "..."};
802 args->callback = &callback_html; 803 args->callback = &callback_html;
803 args->callback_data = (void *) &orig_data; 804 args->callback_data = (void *) &orig_data;
804 return run_query_internal(db, snippet_args, args); 805 return run_query_internal(db, snippet_args, args);
805} 806}
806 807
807/* 808/*
808 * underline a string, pager style. 809 * underline a string, pager style.
809 */ 810 */
810static char * 811static char *
811ul_pager(int ul, const char *s) 812ul_pager(int ul, const char *s)
812{ 813{
813 size_t len; 814 size_t len;
814 char *dst, *d; 815 char *dst, *d;
815 816
816 if (!ul) 817 if (!ul)
817 return estrdup(s); 818 return estrdup(s);
818 819
819 // a -> _\ba 820 // a -> _\ba
820 len = strlen(s) * 3 + 1; 821 len = strlen(s) * 3 + 1;
821 822
822 d = dst = emalloc(len); 823 d = dst = emalloc(len);
823 while (*s) { 824 while (*s) {
824 *d++ = '_'; 825 *d++ = '_';
825 *d++ = '\b'; 826 *d++ = '\b';
826 *d++ = *s++; 827 *d++ = *s++;
827 } 828 }
828 *d = '\0'; 829 *d = '\0';
829 return dst; 830 return dst;
830} 831}
831 832
832/* 833/*
833 * callback_pager -- 834 * callback_pager --
834 * A callback similar to callback_html. It overstrikes the matching text in 835 * A callback similar to callback_html. It overstrikes the matching text in
835 * the snippet so that it appears emboldened when viewed using a pager like 836 * the snippet so that it appears emboldened when viewed using a pager like
836 * more or less. 837 * more or less.
837 */ 838 */
838static int 839static int
839callback_pager(void *data, const char *section, const char *name, 840callback_pager(void *data, const char *section, const char *name,
840 const char *name_desc, const char *snippet, size_t snippet_length) 841 const char *name_desc, const char *snippet, size_t snippet_length)
841{ 842{
842 struct orig_callback_data *orig_data = data; 843 struct orig_callback_data *orig_data = data;
843 char *psnippet; 844 char *psnippet;
844 const char *temp = snippet; 845 const char *temp = snippet;
845 int count = 0; 846 int count = 0;
846 int i = 0, did; 847 int i = 0, did;
847 size_t sz = 0; 848 size_t sz = 0;
848 size_t psnippet_length; 849 size_t psnippet_length;
849 850
850 /* Count the number of bytes of matching text. For each of these 851 /* Count the number of bytes of matching text. For each of these
851 * bytes we will use 2 extra bytes to overstrike it so that it 852 * bytes we will use 2 extra bytes to overstrike it so that it
852 * appears bold when viewed using a pager. 853 * appears bold when viewed using a pager.
853 */ 854 */
854 while (*temp) { 855 while (*temp) {
855 sz = strcspn(temp, "\002\003"); 856 sz = strcspn(temp, "\002\003");
856 temp += sz; 857 temp += sz;
857 if (*temp == '\003') { 858 if (*temp == '\003') {
858 count += 2 * (sz); 859 count += 2 * (sz);
859 } 860 }
860 temp++; 861 temp++;
861 } 862 }
862 863
863 psnippet_length = snippet_length + count; 864 psnippet_length = snippet_length + count;
864 psnippet = emalloc(psnippet_length + 1); 865 psnippet = emalloc(psnippet_length + 1);
865 866
866 /* Copy the bytes from snippet to psnippet: 867 /* Copy the bytes from snippet to psnippet:
867 * 1. Copy the bytes before \002 as it is. 868 * 1. Copy the bytes before \002 as it is.
868 * 2. The bytes after \002 need to be overstriked till we 869 * 2. The bytes after \002 need to be overstriked till we
869 * encounter \003. 870 * encounter \003.
870 * 3. To overstrike a byte 'A' we need to write 'A\bA' 871 * 3. To overstrike a byte 'A' we need to write 'A\bA'
871 */ 872 */
872 did = 0; 873 did = 0;
873 while (*snippet) { 874 while (*snippet) {
874 sz = strcspn(snippet, "\002"); 875 sz = strcspn(snippet, "\002");
875 memcpy(&psnippet[i], snippet, sz); 876 memcpy(&psnippet[i], snippet, sz);
876 snippet += sz; 877 snippet += sz;
877 i += sz; 878 i += sz;
878 879
879 /* Don't change this. Advancing the pointer without reading the byte 880 /* Don't change this. Advancing the pointer without reading the byte
880 * is causing strange behavior. 881 * is causing strange behavior.
881 */ 882 */
882 if (*snippet == '\002') 883 if (*snippet == '\002')
883 snippet++; 884 snippet++;
884 while (*snippet && *snippet != '\003') { 885 while (*snippet && *snippet != '\003') {
885 did = 1; 886 did = 1;
886 psnippet[i++] = *snippet; 887 psnippet[i++] = *snippet;
887 psnippet[i++] = '\b'; 888 psnippet[i++] = '\b';
888 psnippet[i++] = *snippet++; 889 psnippet[i++] = *snippet++;
889 } 890 }
890 if (*snippet) 891 if (*snippet)
891 snippet++; 892 snippet++;
892 } 893 }
893 894
894 psnippet[i] = 0; 895 psnippet[i] = 0;
895 char *ul_section = ul_pager(did, section); 896 char *ul_section = ul_pager(did, section);
896 char *ul_name = ul_pager(did, name); 897 char *ul_name = ul_pager(did, name);
897 char *ul_name_desc = ul_pager(did, name_desc); 898 char *ul_name_desc = ul_pager(did, name_desc);
898 (orig_data->callback)(orig_data->data, ul_section, ul_name, 899 (orig_data->callback)(orig_data->data, ul_section, ul_name,
899 ul_name_desc, psnippet, psnippet_length); 900 ul_name_desc, psnippet, psnippet_length);
900 free(ul_section); 901 free(ul_section);
901 free(ul_name); 902 free(ul_name);
902 free(ul_name_desc); 903 free(ul_name_desc);
903 free(psnippet); 904 free(psnippet);
904 return 0; 905 return 0;
905} 906}
906 907
907struct term_args { 908struct term_args {
908 struct orig_callback_data *orig_data; 909 struct orig_callback_data *orig_data;
909 const char *smul; 910 const char *smul;
910 const char *rmul; 911 const char *rmul;
911}; 912};
912 913
913/* 914/*
914 * underline a string, pager style. 915 * underline a string, pager style.
915 */ 916 */
916static char * 917static char *
917ul_term(const char *s, const struct term_args *ta) 918ul_term(const char *s, const struct term_args *ta)
918{ 919{
919 char *dst; 920 char *dst;
920 921
921 easprintf(&dst, "%s%s%s", ta->smul, s, ta->rmul); 922 easprintf(&dst, "%s%s%s", ta->smul, s, ta->rmul);
922 return dst; 923 return dst;
923} 924}
924 925
925/* 926/*
926 * callback_term -- 927 * callback_term --
927 * A callback similar to callback_html. It overstrikes the matching text in 928 * A callback similar to callback_html. It overstrikes the matching text in
928 * the snippet so that it appears emboldened when viewed using a pager like 929 * the snippet so that it appears emboldened when viewed using a pager like
929 * more or less. 930 * more or less.
930 */ 931 */
931static int 932static int
932callback_term(void *data, const char *section, const char *name, 933callback_term(void *data, const char *section, const char *name,
933 const char *name_desc, const char *snippet, size_t snippet_length) 934 const char *name_desc, const char *snippet, size_t snippet_length)
934{ 935{
935 struct term_args *ta = data; 936 struct term_args *ta = data;
936 struct orig_callback_data *orig_data = ta->orig_data; 937 struct orig_callback_data *orig_data = ta->orig_data;
937 938
938 char *ul_section = ul_term(section, ta); 939 char *ul_section = ul_term(section, ta);
939 char *ul_name = ul_term(name, ta); 940 char *ul_name = ul_term(name, ta);
940 char *ul_name_desc = ul_term(name_desc, ta); 941 char *ul_name_desc = ul_term(name_desc, ta);
941 (orig_data->callback)(orig_data->data, ul_section, ul_name, 942 (orig_data->callback)(orig_data->data, ul_section, ul_name,
942 ul_name_desc, snippet, snippet_length); 943 ul_name_desc, snippet, snippet_length);
943 free(ul_section); 944 free(ul_section);
944 free(ul_name); 945 free(ul_name);
945 free(ul_name_desc); 946 free(ul_name_desc);
946 return 0; 947 return 0;
947} 948}
948 949
949/* 950/*
950 * run_query_pager -- 951 * run_query_pager --
951 * Utility function similar to run_query_html. This function tries to 952 * Utility function similar to run_query_html. This function tries to
952 * pre-process the result assuming it will be piped to a pager. 953 * pre-process the result assuming it will be piped to a pager.
953 * For this purpose it first calls its own callback function callback_pager 954 * For this purpose it first calls its own callback function callback_pager
954 * which then delegates the call to the user supplied callback. 955 * which then delegates the call to the user supplied callback.
955 */ 956 */
956static int 957static int
957run_query_pager(sqlite3 *db, query_args *args) 958run_query_pager(sqlite3 *db, query_args *args)
958{ 959{
959 struct orig_callback_data orig_data; 960 struct orig_callback_data orig_data;
960 orig_data.callback = args->callback; 961 orig_data.callback = args->callback;
961 orig_data.data = args->callback_data; 962 orig_data.data = args->callback_data;
962 const char *snippet_args[3] = { "\002", "\003", "..." }; 963 const char *snippet_args[3] = { "\002", "\003", "..." };
963 args->callback = &callback_pager; 964 args->callback = &callback_pager;
964 args->callback_data = (void *) &orig_data; 965 args->callback_data = (void *) &orig_data;
965 return run_query_internal(db, snippet_args, args); 966 return run_query_internal(db, snippet_args, args);
966} 967}
967 968
968struct nv { 969struct nv {
969 char *s; 970 char *s;
970 size_t l; 971 size_t l;
971}; 972};
972 973
973static int 974static int
974term_putc(int c, void *p) 975term_putc(int c, void *p)
975{ 976{
976 struct nv *nv = p; 977 struct nv *nv = p;
977 nv->s[nv->l++] = c; 978 nv->s[nv->l++] = c;
978 return 0; 979 return 0;
979} 980}
980 981
981static char * 982static char *
982term_fix_seq(TERMINAL *ti, const char *seq) 983term_fix_seq(TERMINAL *ti, const char *seq)
983{ 984{
984 char *res = estrdup(seq); 985 char *res = estrdup(seq);
985 struct nv nv; 986 struct nv nv;
986 987
987 if (ti == NULL) 988 if (ti == NULL)
988 return res; 989 return res;
989 990
990 nv.s = res; 991 nv.s = res;
991 nv.l = 0; 992 nv.l = 0;
992 ti_puts(ti, seq, 1, term_putc, &nv); 993 ti_puts(ti, seq, 1, term_putc, &nv);
993 nv.s[nv.l] = '\0'; 994 nv.s[nv.l] = '\0';
994 995
995 return res; 996 return res;
996} 997}
997 998
998static void 999static void
999term_init(int fd, const char *sa[5]) 1000term_init(int fd, const char *sa[5])
1000{ 1001{
1001 TERMINAL *ti; 1002 TERMINAL *ti;
1002 int error; 1003 int error;
1003 const char *bold, *sgr0, *smso, *rmso, *smul, *rmul; 1004 const char *bold, *sgr0, *smso, *rmso, *smul, *rmul;
1004 1005
1005 if (ti_setupterm(&ti, NULL, fd, &error) == -1) { 1006 if (ti_setupterm(&ti, NULL, fd, &error) == -1) {
1006 bold = sgr0 = NULL; 1007 bold = sgr0 = NULL;
1007 smso = rmso = smul = rmul = ""; 1008 smso = rmso = smul = rmul = "";
1008 ti = NULL; 1009 ti = NULL;
1009 } else { 1010 } else {
1010 bold = ti_getstr(ti, "bold"); 1011 bold = ti_getstr(ti, "bold");
1011 sgr0 = ti_getstr(ti, "sgr0"); 1012 sgr0 = ti_getstr(ti, "sgr0");
1012 if (bold == NULL || sgr0 == NULL) { 1013 if (bold == NULL || sgr0 == NULL) {
1013 smso = ti_getstr(ti, "smso"); 1014 smso = ti_getstr(ti, "smso");
1014 1015
1015 if (smso == NULL || 1016 if (smso == NULL ||
1016 (rmso = ti_getstr(ti, "rmso")) == NULL) 1017 (rmso = ti_getstr(ti, "rmso")) == NULL)
1017 smso = rmso = ""; 1018 smso = rmso = "";
1018 bold = sgr0 = NULL; 1019 bold = sgr0 = NULL;
1019 } else 1020 } else
1020 smso = rmso = ""; 1021 smso = rmso = "";
1021 1022
1022 smul = ti_getstr(ti, "smul"); 1023 smul = ti_getstr(ti, "smul");
1023 if (smul == NULL || (rmul = ti_getstr(ti, "rmul")) == NULL) 1024 if (smul == NULL || (rmul = ti_getstr(ti, "rmul")) == NULL)
1024 smul = rmul = ""; 1025 smul = rmul = "";
1025 } 1026 }
1026 1027
1027 sa[0] = term_fix_seq(ti, bold ? bold : smso); 1028 sa[0] = term_fix_seq(ti, bold ? bold : smso);
1028 sa[1] = term_fix_seq(ti, sgr0 ? sgr0 : rmso); 1029 sa[1] = term_fix_seq(ti, sgr0 ? sgr0 : rmso);
1029 sa[2] = estrdup("..."); 1030 sa[2] = estrdup("...");
1030 sa[3] = term_fix_seq(ti, smul); 1031 sa[3] = term_fix_seq(ti, smul);
1031 sa[4] = term_fix_seq(ti, rmul); 1032 sa[4] = term_fix_seq(ti, rmul);
1032 1033
1033 if (ti) 1034 if (ti)
1034 del_curterm(ti); 1035 del_curterm(ti);
1035} 1036}
1036 1037
1037/* 1038/*
1038 * run_query_term -- 1039 * run_query_term --
1039 * Utility function similar to run_query_html. This function tries to 1040 * Utility function similar to run_query_html. This function tries to
1040 * pre-process the result assuming it will be displayed on a terminal 1041 * pre-process the result assuming it will be displayed on a terminal
1041 * For this purpose it first calls its own callback function callback_pager 1042 * For this purpose it first calls its own callback function callback_pager
1042 * which then delegates the call to the user supplied callback. 1043 * which then delegates the call to the user supplied callback.
1043 */ 1044 */
1044static int 1045static int
1045run_query_term(sqlite3 *db, query_args *args) 1046run_query_term(sqlite3 *db, query_args *args)
1046{ 1047{
1047 struct orig_callback_data orig_data; 1048 struct orig_callback_data orig_data;
1048 struct term_args ta; 1049 struct term_args ta;
1049 orig_data.callback = args->callback; 1050 orig_data.callback = args->callback;
1050 orig_data.data = args->callback_data; 1051 orig_data.data = args->callback_data;
1051 const char *snippet_args[5]; 1052 const char *snippet_args[5];
1052 1053
1053 term_init(STDOUT_FILENO, snippet_args); 1054 term_init(STDOUT_FILENO, snippet_args);
1054 ta.smul = snippet_args[3]; 1055 ta.smul = snippet_args[3];
1055 ta.rmul = snippet_args[4]; 1056 ta.rmul = snippet_args[4];
1056 ta.orig_data = (void *) &orig_data; 1057 ta.orig_data = (void *) &orig_data;
1057 1058
1058 args->callback = &callback_term; 1059 args->callback = &callback_term;
1059 args->callback_data = &ta; 1060 args->callback_data = &ta;
1060 return run_query_internal(db, snippet_args, args); 1061 return run_query_internal(db, snippet_args, args);
1061} 1062}
1062 1063
1063static int 1064static int
1064run_query_none(sqlite3 *db, query_args *args) 1065run_query_none(sqlite3 *db, query_args *args)
1065{ 1066{
1066 struct orig_callback_data orig_data; 1067 struct orig_callback_data orig_data;
1067 orig_data.callback = args->callback; 1068 orig_data.callback = args->callback;
1068 orig_data.data = args->callback_data; 1069 orig_data.data = args->callback_data;
1069 const char *snippet_args[3] = { "", "", "..." }; 1070 const char *snippet_args[3] = { "", "", "..." };
1070 args->callback = &callback_pager; 1071 args->callback = &callback_pager;
1071 args->callback_data = (void *) &orig_data; 1072 args->callback_data = (void *) &orig_data;
1072 return run_query_internal(db, snippet_args, args); 1073 return run_query_internal(db, snippet_args, args);
1073} 1074}
1074 1075
1075int 1076int
1076run_query(sqlite3 *db, query_format fmt, query_args *args) 1077run_query(sqlite3 *db, query_format fmt, query_args *args)
1077{ 1078{
1078 switch (fmt) { 1079 switch (fmt) {
1079 case APROPOS_NONE: 1080 case APROPOS_NONE:
1080 return run_query_none(db, args); 1081 return run_query_none(db, args);
1081 case APROPOS_HTML: 1082 case APROPOS_HTML:
1082 return run_query_html(db, args); 1083 return run_query_html(db, args);
1083 case APROPOS_TERM: 1084 case APROPOS_TERM:
1084 return run_query_term(db, args); 1085 return run_query_term(db, args);
1085 case APROPOS_PAGER: 1086 case APROPOS_PAGER:
1086 return run_query_pager(db, args); 1087 return run_query_pager(db, args);
1087 default: 1088 default:
1088 warnx("Unknown query format %d", (int)fmt); 1089 warnx("Unknown query format %d", (int)fmt);
1089 return -1; 1090 return -1;
1090 } 1091 }
1091} 1092}