test/fts4langid2.test

# 2012 March 01
#
# The author disclaims copyright to this source code.  In place of
# a legal notice, here is a blessing:
#
#    May you do good and not evil.
#    May you find forgiveness for yourself and forgive others.
#    May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library.  The
# focus of this script is testing the languageid=xxx FTS4 option.
#

set testdir [file dirname $argv0]
source $testdir/tester.tcl
set ::testprefix fts4langid2

# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts3 {
  finish_test
  return
}

#-------------------------------------------------------------------------
# Test out-of-range values for the languageid_bits= parameter.
#
do_catchsql_test 1.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=31);
} {1 {languageid_bits parameter out of range}}

do_catchsql_test 1.2 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=-1);
} {1 {languageid_bits parameter out of range}}

do_catchsql_test 1.3 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=0);
  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=30);
} {0 {}}

do_execsql_test 1.4 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Test out-of-range values in the languageid column.
#
do_execsql_test 2.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=8);
  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=7);
}

do_catchsql_test 2.2 {
  INSERT INTO t1(docid, lid, content) VALUES(1, 256, 'abc def');
} {1 {constraint failed}}

do_catchsql_test 2.3 {
  INSERT INTO t2(docid, lid, content) VALUES(1, 128, 'abc def');
} {1 {constraint failed}}

do_catchsql_test 2.3 {
  INSERT INTO t1(docid, lid, content) VALUES(1, -1, 'abc def');
} {1 {constraint failed}}

do_execsql_test 2.4 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Test that if languageid_bits is set to a non-zero value it is
# not possible to specify a non-NULL rowid, even if it is the same
# as the docid.
#
do_execsql_test 3.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=4);
  CREATE VIRTUAL TABLE t2 USING fts4(languageid=lid, languageid_bits=0);
}
  
do_catchsql_test 3.2.1 {
  INSERT INTO t1(rowid, lid, content) VALUES(1, 0, 'abc def');
} {1 {constraint failed}}

do_catchsql_test 3.2.2 {
  INSERT INTO t2(rowid, lid, content) VALUES(1, 0, 'abc def');
} {0 {}}

do_catchsql_test 3.3 {
  INSERT INTO t1(rowid, docid, lid, content) VALUES(2, 2, 0, 'abc def');
} {1 {constraint failed}}

do_catchsql_test 3.4 {
  INSERT INTO t1(lid, content) VALUES(0, 'one two def');
} {1 {constraint failed}}

do_execsql_test 3.4 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Simple tests inserting data with multiple languageid values.
#
do_execsql_test 4.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=5);
}

do_execsql_test 4.2 {
  INSERT INTO t1 (docid, lid, content) VALUES(1, 0, '1 2 3');
  INSERT INTO t1 (docid, lid, content) VALUES(1, 1, '1 2 3 4');
}

do_execsql_test 4.3 {
  SELECT docid, lid FROM t1;
} {1 0 1 1}

do_execsql_test 4.4 {
  SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2';
} {1 0 {1 2 3}}

do_execsql_test 4.5 {
  SELECT docid, lid, content FROM t1 WHERE t1 MATCH '2' AND lid=1;
} {1 1 {1 2 3 4}}

do_execsql_test 4.6 {
  UPDATE t1 SET content = 'x y z' || lid;
  SELECT docid, lid FROM t1;
} {1 0 1 1}

do_execsql_test 3.4 {
  DROP TABLE t1;
}

#-------------------------------------------------------------------------
# Tests for docid range boundary conditions. 
#
for {set bits 1} {$bits <= 30} {incr bits} {
  do_execsql_test 5.$bits.1 "
    CREATE VIRTUAL TABLE t1 USING fts4(languageid=lid, languageid_bits=$bits);
  "

  set max_docid [expr int(1<<(63-$bits))-1]
  set min_docid [expr -1*int(1<<(63-$bits))]
  set max_langid [expr (1<<$bits)-1]
  set min_langid 0


  do_catchsql_test 5.$bits.2.1 {
    INSERT INTO t1(docid, lid, content) VALUES($max_docid+1, 4, '');
  } {1 {constraint failed}}
  do_catchsql_test 5.$bits.2.2 {
    INSERT INTO t1(docid, lid, content) VALUES($min_docid-1, 4, '');
  } {1 {constraint failed}}

  do_test 5.$bits.3 {
    foreach {a b c} "
      $min_docid  $min_langid  {1 min min x}
      $min_docid  $max_langid  {2 min max x}
      $max_docid  $min_langid  {3 max min x}
      $max_docid  $max_langid  {4 max max x}
    " {
      execsql { INSERT INTO t1(docid, lid, content) VALUES($a, $b, $c) }
    }
  } {}

  do_execsql_test 5.$bits.4.1 {
    SELECT docid, lid, content FROM t1 ORDER BY content
  } "
      $min_docid  $min_langid  {1 min min x}
      $min_docid  $max_langid  {2 min max x}
      $max_docid  $min_langid  {3 max min x}
      $max_docid  $max_langid  {4 max max x}
  "

  do_execsql_test 5.$bits.4.2 {
    SELECT docid, lid, content FROM t1 WHERE lid=$min_langid AND t1 MATCH 'x'
  } "
      $min_docid  $min_langid  {1 min min x}
      $max_docid  $min_langid  {3 max min x}
  "

  do_execsql_test 5.$bits.4.3 {
    SELECT docid, lid, content FROM t1 WHERE lid=$max_langid AND t1 MATCH 'x'
  } "
      $min_docid  $max_langid  {2 min max x}
      $max_docid  $max_langid  {4 max max x}
  "

  do_execsql_test 5.$bits.4.4 {
    SELECT docid, lid, content FROM t1 WHERE t1 MATCH '1'
  } "
      $min_docid  $min_langid  {1 min min x}
  "

  do_execsql_test 5.$bits.5 { DROP TABLE t1 }
}

#-------------------------------------------------------------------------
# Tests for auxilliary functions with langaugeid_bits tables.
#
proc mit {blob} {
  set scan(littleEndian) i*
  set scan(bigEndian) I*
  binary scan $blob $scan($::tcl_platform(byteOrder)) r
  return $r
}
db func mit mit

do_execsql_test 6.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=4, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(1, 1, 'one two three four');
  INSERT INTO t1(docid,lid,content) VALUES(2, 1, 'two three four five');
  INSERT INTO t1(docid,lid,content) VALUES(3, 1, 'three four five six');
  INSERT INTO t1(docid,lid,content) VALUES(4, 1, 'four five six seven');

  INSERT INTO t1(docid,lid,content) VALUES(1, 2, 'four three two one');
  INSERT INTO t1(docid,lid,content) VALUES(2, 2, 'five four three two');
  INSERT INTO t1(docid,lid,content) VALUES(3, 2, 'six five four three');
  INSERT INTO t1(docid,lid,content) VALUES(4, 2, 'A B C D');
}

do_execsql_test 6.2.1 {
  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=1;
} {1 {<b>one</b> two three four}}
do_execsql_test 6.2.2 {
  SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'one' AND lid=2;
} {1 {four three two <b>one</b>}}

do_execsql_test 6.2.1 {
  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
} {1 {0 0 4 3} 2 {0 0 0 3}}
do_execsql_test 6.2.2 {
  SELECT docid, offsets(t1) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
} {1 {0 0 11 3} 2 {0 0 16 3}}

do_execsql_test 6.3.1 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=1;
} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
do_execsql_test 6.3.2 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'two' AND lid=2;
} {1 {1 1 1 2 2} 2 {1 1 1 2 2}}
do_execsql_test 6.3.3 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=1;
} {}
do_execsql_test 6.3.4 {
  SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH 'B' AND lid=2;
} {4 {1 1 1 1 1}}

do_execsql_test 6.4 {
  CREATE VIRTUAL TABLE t2 USING fts4(languageid_bits=8, languageid=lid);
  INSERT INTO t2(docid,lid,content) VALUES(-1, 0, 'A B C D');
  INSERT INTO t2(docid,lid,content) VALUES(-2, 0, 'D C B A');
  INSERT INTO t2(docid,lid,content) VALUES(-3, 0, 'C B D A');
  INSERT INTO t2(docid,lid,content) VALUES(-4, 0, 'A D B C');

  INSERT INTO t2(docid,lid,content) VALUES(-1, 1, 'A A A A');
  INSERT INTO t2(docid,lid,content) VALUES(-2, 1, 'B B B B');
  INSERT INTO t2(docid,lid,content) VALUES(-3, 1, 'C C C C');
  INSERT INTO t2(docid,lid,content) VALUES(-4, 1, 'D D D D');
}

do_execsql_test 6.4.1 {
  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B';
} {
  -4 {1 1 1 4 4}
  -3 {1 1 1 4 4}
  -2 {1 1 1 4 4}
  -1 {1 1 1 4 4}
}
do_execsql_test 6.4.2 {
  SELECT docid, mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'B' AND lid=1;
} {-2 {1 1 4 4 1}}

do_execsql_test 6.5 {
  DROP TABLE t1;
  DROP TABLE t2;
}

#-------------------------------------------------------------------------
# Tests for querying by docid.
#
do_execsql_test 7.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=8, languageid=lid);
  INSERT INTO t1(docid,lid,content) VALUES(10, 10, 'abc def');
}

do_execsql_test 7.2 {
  SELECT docid,lid,content FROM t1 WHERE docid=10;
} {10 10 {abc def}}

do_execsql_test 7.3 {
  SELECT docid,lid,content FROM t1 WHERE docid<11;
} {10 10 {abc def}}

do_execsql_test 7.4 {
  DROP TABLE t1;
}

#-------------------------------------------------------------------------
# Tests for sorting by docid.
#
do_execsql_test 8.1 {
  CREATE VIRTUAL TABLE t1 USING fts4(languageid_bits=6, languageid=lid);
  INSERT INTO t1 (docid,lid,content) VALUES(1, 0, 'abc def');
  INSERT INTO t1 (docid,lid,content) VALUES(3, 0, 'abc ghi');
  INSERT INTO t1 (docid,lid,content) VALUES(2, 0, 'def ghi');

  INSERT INTO t1 (docid,lid,content) VALUES(1, 5, 'A B');
  INSERT INTO t1 (docid,lid,content) VALUES(3, 5, 'A C');
  INSERT INTO t1 (docid,lid,content) VALUES(2, 5, 'B C');
}

do_execsql_test 8.2 {
  SELECT docid FROM t1 ORDER BY docid;
} {1 1 2 2 3 3}
do_execsql_test 8.3 {
  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid;
} {2 3}
do_execsql_test 8.4 {
  SELECT docid FROM t1 WHERE t1 MATCH 'ghi' ORDER BY docid DESC;
} {3 2}

# Test that the docid and languageid fields may be updated.
#
do_execsql_test 8.5 { UPDATE t1 SET docid=docid+3, lid=0 WHERE lid=5; }
do_execsql_test 8.6 { SELECT docid FROM t1 ORDER BY docid; } {1 2 3 4 5 6}
do_execsql_test 8.7 { SELECT docid FROM t1 WHERE t1 MATCH 'A' } {4 6}
do_execsql_test 8.8 { SELECT docid FROM t1 WHERE t1 MATCH 'A' AND lid=5 } {}

finish_test