http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/core/Lucy/Test/Util/TestNumberUtils.c ---------------------------------------------------------------------- diff --git a/core/Lucy/Test/Util/TestNumberUtils.c b/core/Lucy/Test/Util/TestNumberUtils.c deleted file mode 100644 index f9cf3a1..0000000 --- a/core/Lucy/Test/Util/TestNumberUtils.c +++ /dev/null @@ -1,473 +0,0 @@ -/* Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdlib.h> -#include <time.h> - -#define TESTLUCY_USE_SHORT_NAMES -#include "Lucy/Util/ToolSet.h" - -#include "charmony.h" - -#include "Lucy/Test/Util/TestNumberUtils.h" - -#include "Clownfish/TestHarness/TestBatchRunner.h" -#include "Clownfish/TestHarness/TestUtils.h" -#include "Lucy/Util/NumberUtils.h" - -TestNumberUtils* -TestNumUtil_new() { - return (TestNumberUtils*)Class_Make_Obj(TESTNUMBERUTILS); -} - -static void -test_u1(TestBatchRunner *runner) { - size_t count = 64; - uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, 2); - size_t amount = count / 8; - uint8_t *bits = (uint8_t*)CALLOCATE(amount, sizeof(uint8_t)); - - for (size_t i = 0; i < count; i++) { - if (ints[i]) { NumUtil_u1set(bits, i); } - } - for (size_t i = 0; i < count; i++) { - TEST_UINT_EQ(runner, NumUtil_u1get(bits, i), ints[i], "u1 set/get"); - } - - for (size_t i = 0; i < count; i++) { - NumUtil_u1flip(bits, i); - } - for (size_t i = 0; i < count; i++) { - TEST_UINT_EQ(runner, NumUtil_u1get(bits, i), !ints[i], "u1 flip"); - } - - FREEMEM(bits); - FREEMEM(ints); -} - -static void -test_u2(TestBatchRunner *runner) { - size_t count = 32; - uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, 4); - uint8_t *bits = (uint8_t*)CALLOCATE((count / 4), sizeof(uint8_t)); - - for (size_t i = 0; i < count; i++) { - NumUtil_u2set(bits, i, (uint8_t)ints[i]); - } - for (size_t i = 0; i < count; i++) { - TEST_UINT_EQ(runner, NumUtil_u2get(bits, i), ints[i], "u2"); - } - - FREEMEM(bits); - FREEMEM(ints); -} - -static void -test_u4(TestBatchRunner *runner) { - size_t count = 128; - uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, 16); - uint8_t *bits = (uint8_t*)CALLOCATE((count / 2), sizeof(uint8_t)); - - for (size_t i = 0; i < count; i++) { - NumUtil_u4set(bits, i, (uint8_t)ints[i]); - } - for (size_t i = 0; i < count; i++) { - TEST_UINT_EQ(runner, NumUtil_u4get(bits, i), ints[i], "u4"); - } - - FREEMEM(bits); - FREEMEM(ints); -} - -static void -test_ci32(TestBatchRunner *runner) { - int64_t mins[] = { -500, -0x4000 - 100, INT32_MIN }; - int64_t limits[] = { 500, -0x4000 + 100, INT32_MIN + 10 }; - int32_t set_num; - int32_t num_sets = sizeof(mins) / sizeof(int64_t); - size_t count = 64; - int64_t *ints = NULL; - size_t amount = count * CI32_MAX_BYTES; - char *encoded = (char*)CALLOCATE(amount, sizeof(char)); - char *target = encoded; - char *limit = target + amount; - const char *decode; - - for (set_num = 0; set_num < num_sets; set_num++) { - const char *skip; - ints = TestUtils_random_i64s(ints, count, - mins[set_num], limits[set_num]); - target = encoded; - for (size_t i = 0; i < count; i++) { - NumUtil_encode_ci32((int32_t)ints[i], &target); - } - decode = encoded; - skip = encoded; - for (size_t i = 0; i < count; i++) { - TEST_INT_EQ(runner, NumUtil_decode_ci32(&decode), ints[i], - "ci32 %" PRId64, ints[i]); - NumUtil_skip_cint(&skip); - if (decode > limit) { THROW(ERR, "overrun"); } - } - TEST_TRUE(runner, skip == decode, "skip %p == %p", skip, decode); - } - - target = encoded; - NumUtil_encode_ci32(INT32_MAX, &target); - decode = encoded; - TEST_INT_EQ(runner, NumUtil_decode_ci32(&decode), INT32_MAX, - "ci32 INT32_MAX"); - target = encoded; - NumUtil_encode_ci32(INT32_MIN, &target); - decode = encoded; - TEST_INT_EQ(runner, NumUtil_decode_ci32(&decode), INT32_MIN, - "ci32 INT32_MIN"); - - FREEMEM(encoded); - FREEMEM(ints); -} - -static void -test_cu32(TestBatchRunner *runner) { - uint64_t mins[] = { 0, 0x4000 - 100, (uint32_t)INT32_MAX - 100, UINT32_MAX - 10 }; - uint64_t limits[] = { 500, 0x4000 + 100, (uint32_t)INT32_MAX + 100, UINT32_MAX }; - uint32_t set_num; - uint32_t num_sets = sizeof(mins) / sizeof(uint64_t); - size_t count = 64; - uint64_t *ints = NULL; - size_t amount = count * CU32_MAX_BYTES; - char *encoded = (char*)CALLOCATE(amount, sizeof(char)); - char *target = encoded; - char *limit = target + amount; - const char *decode; - - for (set_num = 0; set_num < num_sets; set_num++) { - const char *skip; - ints = TestUtils_random_u64s(ints, count, - mins[set_num], limits[set_num]); - target = encoded; - for (size_t i = 0; i < count; i++) { - ints[i] = (uint32_t)ints[i]; - NumUtil_encode_cu32((uint32_t)ints[i], &target); - } - decode = encoded; - skip = encoded; - for (size_t i = 0; i < count; i++) { - TEST_UINT_EQ(runner, NumUtil_decode_cu32(&decode), ints[i], - "cu32 %lu", (unsigned long)ints[i]); - NumUtil_skip_cint(&skip); - if (decode > limit) { THROW(ERR, "overrun"); } - } - TEST_TRUE(runner, skip == decode, "skip %p == %p", skip, decode); - - target = encoded; - for (size_t i = 0; i < count; i++) { - NumUtil_encode_padded_cu32((uint32_t)ints[i], &target); - } - TEST_TRUE(runner, target == limit, - "padded cu32 uses 5 bytes (%p == %p)", target, limit); - decode = encoded; - skip = encoded; - for (size_t i = 0; i < count; i++) { - TEST_UINT_EQ(runner, NumUtil_decode_cu32(&decode), ints[i], - "padded cu32 %lu", (unsigned long)ints[i]); - NumUtil_skip_cint(&skip); - if (decode > limit) { THROW(ERR, "overrun"); } - } - TEST_TRUE(runner, skip == decode, "skip padded %p == %p", skip, - decode); - } - - target = encoded; - NumUtil_encode_cu32(UINT32_MAX, &target); - decode = encoded; - TEST_UINT_EQ(runner, NumUtil_decode_cu32(&decode), UINT32_MAX, "cu32 UINT32_MAX"); - - FREEMEM(encoded); - FREEMEM(ints); -} - -static void -test_ci64(TestBatchRunner *runner) { - int64_t mins[] = { -500, -0x4000 - 100, (int64_t)INT32_MIN - 100, INT64_MIN }; - int64_t limits[] = { 500, -0x4000 + 100, (int64_t)INT32_MIN + 1000, INT64_MIN + 10 }; - int32_t set_num; - int32_t num_sets = sizeof(mins) / sizeof(int64_t); - size_t count = 64; - int64_t *ints = NULL; - size_t amount = count * CI64_MAX_BYTES; - char *encoded = (char*)CALLOCATE(amount, sizeof(char)); - char *target = encoded; - char *limit = target + amount; - const char *decode; - - for (set_num = 0; set_num < num_sets; set_num++) { - const char *skip; - ints = TestUtils_random_i64s(ints, count, - mins[set_num], limits[set_num]); - target = encoded; - for (size_t i = 0; i < count; i++) { - NumUtil_encode_ci64(ints[i], &target); - } - decode = encoded; - skip = encoded; - for (size_t i = 0; i < count; i++) { - int64_t got = NumUtil_decode_ci64(&decode); - TEST_INT_EQ(runner, got, ints[i], - "ci64 %" PRId64 " == %" PRId64, got, ints[i]); - if (decode > limit) { THROW(ERR, "overrun"); } - NumUtil_skip_cint(&skip); - } - TEST_TRUE(runner, skip == decode, "skip %p == %p", skip, decode); - } - - target = encoded; - NumUtil_encode_ci64(INT64_MAX, &target); - decode = encoded; - int64_t got = NumUtil_decode_ci64(&decode); - TEST_INT_EQ(runner, got, INT64_MAX, "ci64 INT64_MAX"); - - target = encoded; - NumUtil_encode_ci64(INT64_MIN, &target); - decode = encoded; - got = NumUtil_decode_ci64(&decode); - TEST_INT_EQ(runner, got, INT64_MIN, "ci64 INT64_MIN"); - - FREEMEM(encoded); - FREEMEM(ints); -} - -static void -test_cu64(TestBatchRunner *runner) { - uint64_t mins[] = { 0, 0x4000 - 100, (uint64_t)UINT32_MAX - 100, UINT64_MAX - 10 }; - uint64_t limits[] = { 500, 0x4000 + 100, (uint64_t)UINT32_MAX + 1000, UINT64_MAX }; - uint32_t set_num; - uint32_t num_sets = sizeof(mins) / sizeof(uint64_t); - size_t count = 64; - uint64_t *ints = NULL; - size_t amount = count * CU64_MAX_BYTES; - char *encoded = (char*)CALLOCATE(amount, sizeof(char)); - char *target = encoded; - char *limit = target + amount; - const char *decode; - - for (set_num = 0; set_num < num_sets; set_num++) { - const char *skip; - ints = TestUtils_random_u64s(ints, count, - mins[set_num], limits[set_num]); - target = encoded; - for (size_t i = 0; i < count; i++) { - NumUtil_encode_cu64(ints[i], &target); - } - decode = encoded; - skip = encoded; - for (size_t i = 0; i < count; i++) { - uint64_t got = NumUtil_decode_cu64(&decode); - TEST_TRUE(runner, got == ints[i], - "cu64 %" PRIu64 " == %" PRIu64, got, ints[i]); - if (decode > limit) { THROW(ERR, "overrun"); } - NumUtil_skip_cint(&skip); - } - TEST_TRUE(runner, skip == decode, "skip %p == %p", skip, decode); - } - - target = encoded; - NumUtil_encode_cu64(UINT64_MAX, &target); - - decode = encoded; - uint64_t got = NumUtil_decode_cu64(&decode); - TEST_TRUE(runner, got == UINT64_MAX, "cu64 UINT64_MAX"); - - FREEMEM(encoded); - FREEMEM(ints); -} - -static void -test_bigend_u16(TestBatchRunner *runner) { - size_t count = 32; - uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, UINT16_MAX + 1); - size_t amount = (count + 1) * sizeof(uint16_t); - char *allocated = (char*)CALLOCATE(amount, sizeof(char)); - char *encoded = allocated + 1; // Intentionally misaligned. - char *target = encoded; - - for (size_t i = 0; i < count; i++) { - NumUtil_encode_bigend_u16((uint16_t)ints[i], &target); - target += sizeof(uint16_t); - } - target = encoded; - for (size_t i = 0; i < count; i++) { - uint16_t got = NumUtil_decode_bigend_u16(target); - TEST_INT_EQ(runner, got, (long)ints[i], "bigend u16"); - target += sizeof(uint16_t); - } - - target = encoded; - NumUtil_encode_bigend_u16(1, &target); - TEST_INT_EQ(runner, encoded[0], 0, "Truly big-endian u16"); - TEST_INT_EQ(runner, encoded[1], 1, "Truly big-endian u16"); - - FREEMEM(allocated); - FREEMEM(ints); -} - -static void -test_bigend_u32(TestBatchRunner *runner) { - size_t count = 32; - uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, UINT64_C(1) + UINT32_MAX); - size_t amount = (count + 1) * sizeof(uint32_t); - char *allocated = (char*)CALLOCATE(amount, sizeof(char)); - char *encoded = allocated + 1; // Intentionally misaligned. - char *target = encoded; - - for (size_t i = 0; i < count; i++) { - ints[i] = (uint32_t)ints[i]; - NumUtil_encode_bigend_u32((uint32_t)ints[i], &target); - target += sizeof(uint32_t); - } - target = encoded; - for (size_t i = 0; i < count; i++) { - uint32_t got = NumUtil_decode_bigend_u32(target); - TEST_UINT_EQ(runner, got, ints[i], "bigend u32"); - target += sizeof(uint32_t); - } - - target = encoded; - NumUtil_encode_bigend_u32(1, &target); - TEST_INT_EQ(runner, encoded[0], 0, "Truly big-endian u32"); - TEST_INT_EQ(runner, encoded[3], 1, "Truly big-endian u32"); - - FREEMEM(allocated); - FREEMEM(ints); -} - -static void -test_bigend_u64(TestBatchRunner *runner) { - size_t count = 32; - uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, UINT64_MAX); - size_t amount = (count + 1) * sizeof(uint64_t); - char *allocated = (char*)CALLOCATE(amount, sizeof(char)); - char *encoded = allocated + 1; // Intentionally misaligned. - char *target = encoded; - - for (size_t i = 0; i < count; i++) { - NumUtil_encode_bigend_u64(ints[i], &target); - target += sizeof(uint64_t); - } - target = encoded; - for (size_t i = 0; i < count; i++) { - uint64_t got = NumUtil_decode_bigend_u64(target); - TEST_TRUE(runner, got == ints[i], "bigend u64"); - target += sizeof(uint64_t); - } - - target = encoded; - NumUtil_encode_bigend_u64(1, &target); - TEST_INT_EQ(runner, encoded[0], 0, "Truly big-endian"); - TEST_INT_EQ(runner, encoded[7], 1, "Truly big-endian"); - - FREEMEM(allocated); - FREEMEM(ints); -} - -static void -test_bigend_f32(TestBatchRunner *runner) { - float source[] = { -1.3f, 0.0f, 100.2f }; - size_t count = 3; - size_t amount = (count + 1) * sizeof(float); - uint8_t *allocated = (uint8_t*)CALLOCATE(amount, sizeof(uint8_t)); - uint8_t *encoded = allocated + 1; // Intentionally misaligned. - uint8_t *target = encoded; - - for (size_t i = 0; i < count; i++) { - NumUtil_encode_bigend_f32(source[i], &target); - target += sizeof(float); - } - target = encoded; - for (size_t i = 0; i < count; i++) { - float got = NumUtil_decode_bigend_f32(target); - TEST_TRUE(runner, got == source[i], "bigend f32"); - target += sizeof(float); - } - - target = encoded; - NumUtil_encode_bigend_f32(-2.0f, &target); - TEST_INT_EQ(runner, (encoded[0] & 0x80), 0x80, - "Truly big-endian (IEEE 754 sign bit set for negative number)"); - TEST_INT_EQ(runner, encoded[0], 0xC0, - "IEEE 754 representation of -2.0f, byte 0"); - for (size_t i = 1; i < sizeof(float); i++) { - TEST_INT_EQ(runner, encoded[i], 0, - "IEEE 754 representation of -2.0f, byte %d", (int)i); - } - - FREEMEM(allocated); -} - -static void -test_bigend_f64(TestBatchRunner *runner) { - double source[] = { -1.3, 0.0, 100.2 }; - size_t count = 3; - size_t amount = (count + 1) * sizeof(double); - uint8_t *allocated = (uint8_t*)CALLOCATE(amount, sizeof(uint8_t)); - uint8_t *encoded = allocated + 1; // Intentionally misaligned. - uint8_t *target = encoded; - - for (size_t i = 0; i < count; i++) { - NumUtil_encode_bigend_f64(source[i], &target); - target += sizeof(double); - } - target = encoded; - for (size_t i = 0; i < count; i++) { - double got = NumUtil_decode_bigend_f64(target); - TEST_TRUE(runner, got == source[i], "bigend f64"); - target += sizeof(double); - } - - target = encoded; - NumUtil_encode_bigend_f64(-2.0, &target); - TEST_INT_EQ(runner, (encoded[0] & 0x80), 0x80, - "Truly big-endian (IEEE 754 sign bit set for negative number)"); - TEST_INT_EQ(runner, encoded[0], 0xC0, - "IEEE 754 representation of -2.0, byte 0"); - for (size_t i = 1; i < sizeof(double); i++) { - TEST_INT_EQ(runner, encoded[i], 0, - "IEEE 754 representation of -2.0, byte %d", (int)i); - } - - FREEMEM(allocated); -} - -void -TestNumUtil_Run_IMP(TestNumberUtils *self, TestBatchRunner *runner) { - TestBatchRunner_Plan(runner, (TestBatch*)self, 1655); - srand((unsigned int)time((time_t*)NULL)); - test_u1(runner); - test_u2(runner); - test_u4(runner); - test_ci32(runner); - test_cu32(runner); - test_ci64(runner); - test_cu64(runner); - test_bigend_u16(runner); - test_bigend_u32(runner); - test_bigend_u64(runner); - test_bigend_f32(runner); - test_bigend_f64(runner); -} - - -
http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/core/Lucy/Test/Util/TestNumberUtils.cfh ---------------------------------------------------------------------- diff --git a/core/Lucy/Test/Util/TestNumberUtils.cfh b/core/Lucy/Test/Util/TestNumberUtils.cfh deleted file mode 100644 index fcb20a9..0000000 --- a/core/Lucy/Test/Util/TestNumberUtils.cfh +++ /dev/null @@ -1,29 +0,0 @@ -/* Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -parcel TestLucy; - -class Lucy::Test::Util::TestNumberUtils nickname TestNumUtil - inherits Clownfish::TestHarness::TestBatch { - - inert incremented TestNumberUtils* - new(); - - void - Run(TestNumberUtils *self, TestBatchRunner *runner); -} - - http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/core/Lucy/Test/Util/TestPriorityQueue.c ---------------------------------------------------------------------- diff --git a/core/Lucy/Test/Util/TestPriorityQueue.c b/core/Lucy/Test/Util/TestPriorityQueue.c deleted file mode 100644 index 7265170..0000000 --- a/core/Lucy/Test/Util/TestPriorityQueue.c +++ /dev/null @@ -1,165 +0,0 @@ -/* Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#define C_TESTLUCY_TESTPRIORITYQUEUE -#define TESTLUCY_USE_SHORT_NAMES -#include "Lucy/Util/ToolSet.h" - -#include "Clownfish/Num.h" -#include "Clownfish/TestHarness/TestBatchRunner.h" -#include "Lucy/Test.h" -#include "Lucy/Test/Util/TestPriorityQueue.h" -#include "Lucy/Util/PriorityQueue.h" - -TestPriorityQueue* -TestPriQ_new() { - return (TestPriorityQueue*)Class_Make_Obj(TESTPRIORITYQUEUE); -} - -NumPriorityQueue* -NumPriQ_new(uint32_t max_size) { - NumPriorityQueue *self - = (NumPriorityQueue*)Class_Make_Obj(NUMPRIORITYQUEUE); - return (NumPriorityQueue*)PriQ_init((PriorityQueue*)self, max_size); -} - -bool -NumPriQ_Less_Than_IMP(NumPriorityQueue *self, Obj *a, Obj *b) { - Float *num_a = (Float*)a; - Float *num_b = (Float*)b; - UNUSED_VAR(self); - return Float_Get_Value(num_a) < Float_Get_Value(num_b) ? true : false; -} - -static void -S_insert_num(NumPriorityQueue *pq, int32_t value) { - NumPriQ_Insert(pq, (Obj*)Float_new((double)value)); -} - -static int32_t -S_pop_num(NumPriorityQueue *pq) { - Float *num = (Float*)NumPriQ_Pop(pq); - int32_t retval; - if (!num) { THROW(ERR, "Queue is empty"); } - retval = (int32_t)Float_Get_Value(num); - DECREF(num); - return retval; -} - -static void -test_Peek_and_Pop_All(TestBatchRunner *runner) { - NumPriorityQueue *pq = NumPriQ_new(5); - Float *val; - - S_insert_num(pq, 3); - S_insert_num(pq, 1); - S_insert_num(pq, 2); - S_insert_num(pq, 20); - S_insert_num(pq, 10); - val = (Float*)CERTIFY(NumPriQ_Peek(pq), FLOAT); - TEST_INT_EQ(runner, (long)Float_Get_Value(val), 1, - "peek at the least item in the queue"); - - Vector *got = NumPriQ_Pop_All(pq); - val = (Float*)CERTIFY(Vec_Fetch(got, 0), FLOAT); - TEST_INT_EQ(runner, (long)Float_Get_Value(val), 20, "pop_all"); - val = (Float*)CERTIFY(Vec_Fetch(got, 1), FLOAT); - TEST_INT_EQ(runner, (long)Float_Get_Value(val), 10, "pop_all"); - val = (Float*)CERTIFY(Vec_Fetch(got, 2), FLOAT); - TEST_INT_EQ(runner, (long)Float_Get_Value(val), 3, "pop_all"); - val = (Float*)CERTIFY(Vec_Fetch(got, 3), FLOAT); - TEST_INT_EQ(runner, (long)Float_Get_Value(val), 2, "pop_all"); - val = (Float*)CERTIFY(Vec_Fetch(got, 4), FLOAT); - TEST_INT_EQ(runner, (long)Float_Get_Value(val), 1, "pop_all"); - - DECREF(got); - DECREF(pq); -} - -static void -test_Insert_and_Pop(TestBatchRunner *runner) { - NumPriorityQueue *pq = NumPriQ_new(5); - - S_insert_num(pq, 3); - S_insert_num(pq, 1); - S_insert_num(pq, 2); - S_insert_num(pq, 20); - S_insert_num(pq, 10); - - TEST_INT_EQ(runner, S_pop_num(pq), 1, "Pop"); - TEST_INT_EQ(runner, S_pop_num(pq), 2, "Pop"); - TEST_INT_EQ(runner, S_pop_num(pq), 3, "Pop"); - TEST_INT_EQ(runner, S_pop_num(pq), 10, "Pop"); - - S_insert_num(pq, 7); - TEST_INT_EQ(runner, S_pop_num(pq), 7, - "Insert after Pop still sorts correctly"); - - DECREF(pq); -} - -static void -test_discard(TestBatchRunner *runner) { - int32_t i; - NumPriorityQueue *pq = NumPriQ_new(5); - - for (i = 1; i <= 10; i++) { S_insert_num(pq, i); } - S_insert_num(pq, -3); - for (i = 1590; i <= 1600; i++) { S_insert_num(pq, i); } - S_insert_num(pq, 5); - - TEST_INT_EQ(runner, S_pop_num(pq), 1596, "discard waste"); - TEST_INT_EQ(runner, S_pop_num(pq), 1597, "discard waste"); - TEST_INT_EQ(runner, S_pop_num(pq), 1598, "discard waste"); - TEST_INT_EQ(runner, S_pop_num(pq), 1599, "discard waste"); - TEST_INT_EQ(runner, S_pop_num(pq), 1600, "discard waste"); - - DECREF(pq); -} - -static void -test_random_insertion(TestBatchRunner *runner) { - int i; - int shuffled[64]; - NumPriorityQueue *pq = NumPriQ_new(64); - - for (i = 0; i < 64; i++) { shuffled[i] = i; } - for (i = 0; i < 64; i++) { - int shuffle_pos = rand() % 64; - int temp = shuffled[shuffle_pos]; - shuffled[shuffle_pos] = shuffled[i]; - shuffled[i] = temp; - } - for (i = 0; i < 64; i++) { S_insert_num(pq, shuffled[i]); } - for (i = 0; i < 64; i++) { - if (S_pop_num(pq) != i) { break; } - } - TEST_INT_EQ(runner, i, 64, "random insertion"); - - DECREF(pq); -} - -void -TestPriQ_Run_IMP(TestPriorityQueue *self, TestBatchRunner *runner) { - TestBatchRunner_Plan(runner, (TestBatch*)self, 17); - test_Peek_and_Pop_All(runner); - test_Insert_and_Pop(runner); - test_discard(runner); - test_random_insertion(runner); -} - - - http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/core/Lucy/Test/Util/TestPriorityQueue.cfh ---------------------------------------------------------------------- diff --git a/core/Lucy/Test/Util/TestPriorityQueue.cfh b/core/Lucy/Test/Util/TestPriorityQueue.cfh deleted file mode 100644 index 4f434f1..0000000 --- a/core/Lucy/Test/Util/TestPriorityQueue.cfh +++ /dev/null @@ -1,39 +0,0 @@ -/* Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -parcel TestLucy; - -class Lucy::Test::Util::NumPriorityQueue nickname NumPriQ - inherits Lucy::Util::PriorityQueue { - - inert incremented NumPriorityQueue* - new(uint32_t max_size); - - bool - Less_Than(NumPriorityQueue *self, Obj *a, Obj *b); -} - -class Lucy::Test::Util::TestPriorityQueue nickname TestPriQ - inherits Clownfish::TestHarness::TestBatch { - - inert incremented TestPriorityQueue* - new(); - - void - Run(TestPriorityQueue *self, TestBatchRunner *runner); -} - - http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/core/Lucy/Test/Util/TestSortExternal.c ---------------------------------------------------------------------- diff --git a/core/Lucy/Test/Util/TestSortExternal.c b/core/Lucy/Test/Util/TestSortExternal.c deleted file mode 100644 index b55dcc2..0000000 --- a/core/Lucy/Test/Util/TestSortExternal.c +++ /dev/null @@ -1,324 +0,0 @@ -/* Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stdlib.h> -#include <string.h> -#include <time.h> - -#define TESTLUCY_USE_SHORT_NAMES -#include "Lucy/Util/ToolSet.h" -#include "Lucy/Test/Util/TestSortExternal.h" - -#include "Clownfish/Blob.h" -#include "Clownfish/TestHarness/TestBatchRunner.h" -#include "Lucy/Util/BlobSortEx.h" -#include "Lucy/Util/SortExternal.h" - -static Blob *a_blob; -static Blob *b_blob; -static Blob *c_blob; -static Blob *d_blob; -static Blob *x_blob; -static Blob *y_blob; -static Blob *z_blob; - -TestSortExternal* -TestSortExternal_new() { - return (TestSortExternal*)Class_Make_Obj(TESTSORTEXTERNAL); -} - -static void -S_init_blobs() { - a_blob = Blob_new("a", 1); - b_blob = Blob_new("b", 1); - c_blob = Blob_new("c", 1); - d_blob = Blob_new("d", 1); - x_blob = Blob_new("x", 1); - y_blob = Blob_new("y", 1); - z_blob = Blob_new("z", 1); -} - -static void -S_destroy_blobs() { - DECREF(a_blob); - DECREF(b_blob); - DECREF(c_blob); - DECREF(d_blob); - DECREF(x_blob); - DECREF(y_blob); - DECREF(z_blob); -} - -static void -test_bbsortex(TestBatchRunner *runner) { - BlobSortEx *sortex = BlobSortEx_new(4, NULL); - - BlobSortEx_Feed(sortex, INCREF(c_blob)); - TEST_INT_EQ(runner, BlobSortEx_Buffer_Count(sortex), 1, - "feed elem into cache"); - - BlobSortEx_Feed(sortex, INCREF(b_blob)); - BlobSortEx_Feed(sortex, INCREF(d_blob)); - BlobSortEx_Sort_Buffer(sortex); - - { - Vector *cache = BlobSortEx_Peek_Cache(sortex); - Vector *wanted = Vec_new(3); - Vec_Push(wanted, INCREF(b_blob)); - Vec_Push(wanted, INCREF(c_blob)); - Vec_Push(wanted, INCREF(d_blob)); - TEST_TRUE(runner, Vec_Equals(cache, (Obj*)wanted), "sort cache"); - DECREF(wanted); - DECREF(cache); - } - - BlobSortEx_Feed(sortex, INCREF(a_blob)); - TEST_INT_EQ(runner, BlobSortEx_Buffer_Count(sortex), 0, - "cache flushed automatically when mem_thresh crossed"); - TEST_INT_EQ(runner, BlobSortEx_Get_Num_Runs(sortex), 1, "run added"); - - Vector *external = Vec_new(3); - Vec_Push(external, INCREF(x_blob)); - Vec_Push(external, INCREF(y_blob)); - Vec_Push(external, INCREF(z_blob)); - BlobSortEx *run = BlobSortEx_new(0x1000000, external); - BlobSortEx_Add_Run(sortex, (SortExternal*)run); - BlobSortEx_Flip(sortex); - - { - Vector *got = Vec_new(7); - Obj *object; - while (NULL != (object = BlobSortEx_Fetch(sortex))) { - Vec_Push(got, object); - } - - Vector *wanted = Vec_new(7); - Vec_Push(wanted, INCREF(a_blob)); - Vec_Push(wanted, INCREF(b_blob)); - Vec_Push(wanted, INCREF(c_blob)); - Vec_Push(wanted, INCREF(d_blob)); - Vec_Push(wanted, INCREF(x_blob)); - Vec_Push(wanted, INCREF(y_blob)); - Vec_Push(wanted, INCREF(z_blob)); - - TEST_TRUE(runner, Vec_Equals(got, (Obj*)wanted), "Add_Run"); - - DECREF(wanted); - DECREF(got); - } - - DECREF(external); - DECREF(sortex); -} - -static void -test_clear_buffer(TestBatchRunner *runner) { - BlobSortEx *sortex = BlobSortEx_new(4, NULL); - - BlobSortEx_Feed(sortex, INCREF(c_blob)); - BlobSortEx_Clear_Buffer(sortex); - TEST_INT_EQ(runner, BlobSortEx_Buffer_Count(sortex), 0, "Clear_Buffer"); - - BlobSortEx_Feed(sortex, INCREF(b_blob)); - BlobSortEx_Feed(sortex, INCREF(a_blob)); - BlobSortEx_Flush(sortex); - BlobSortEx_Flip(sortex); - Obj *object = BlobSortEx_Peek(sortex); - TEST_TRUE(runner, Blob_Equals(a_blob, object), "Peek"); - - Vector *got = Vec_new(2); - while (NULL != (object = BlobSortEx_Fetch(sortex))) { - Vec_Push(got, object); - } - Vector *wanted = Vec_new(2); - Vec_Push(wanted, INCREF(a_blob)); - Vec_Push(wanted, INCREF(b_blob)); - TEST_TRUE(runner, Vec_Equals(got, (Obj*)wanted), - "elements cleared via Clear_Buffer truly cleared"); - - DECREF(wanted); - DECREF(got); - DECREF(sortex); -} - -static void -S_test_sort(TestBatchRunner *runner, Vector *blobs, uint32_t mem_thresh, - const char *test_name) { - size_t size = Vec_Get_Size(blobs); - BlobSortEx *sortex = BlobSortEx_new(mem_thresh, NULL); - Blob **shuffled = (Blob**)MALLOCATE(size * sizeof(Blob*)); - - for (size_t i = 0; i < size; ++i) { - shuffled[i] = (Blob*)CERTIFY(Vec_Fetch(blobs, i), BLOB); - } - for (int i = (int)size - 1; i > 0; --i) { - int shuffle_pos = rand() % (i + 1); - Blob *temp = shuffled[shuffle_pos]; - shuffled[shuffle_pos] = shuffled[i]; - shuffled[i] = temp; - } - for (size_t i = 0; i < size; ++i) { - BlobSortEx_Feed(sortex, INCREF(shuffled[i])); - } - - BlobSortEx_Flip(sortex); - Vector *got = Vec_new(size); - Obj *object; - while (NULL != (object = BlobSortEx_Fetch(sortex))) { - Vec_Push(got, object); - } - TEST_TRUE(runner, Vec_Equals(got, (Obj*)blobs), test_name); - - FREEMEM(shuffled); - DECREF(got); - DECREF(sortex); -} - -static void -S_test_sort_letters(TestBatchRunner *runner, const char *letters, - uint32_t mem_thresh, const char *test_name) { - size_t num_letters = strlen(letters); - Vector *blobs = Vec_new(num_letters); - - for (size_t i = 0; i < num_letters; ++i) { - char ch = letters[i]; - size_t size = ch == '_' ? 0 : 1; - Blob *blob = Blob_new(&ch, size); - Vec_Push(blobs, (Obj*)blob); - } - - S_test_sort(runner, blobs, mem_thresh, test_name); - - DECREF(blobs); -} - -static void -test_sort_letters(TestBatchRunner *runner) { - S_test_sort_letters(runner, "abcdefghijklmnopqrstuvwxyz", 0x1000000, - "sort letters"); - S_test_sort_letters(runner, "aaabcdxxxxxxyy", 0x1000000, - "sort repeated letters"); - S_test_sort_letters(runner, "__abcdefghijklmnopqrstuvwxyz", 0x1000000, - "sort letters and empty strings"); - S_test_sort_letters(runner, "abcdefghijklmnopqrstuvwxyz", 30, - "... with an absurdly low mem_thresh"); - S_test_sort_letters(runner, "abcdefghijklmnopqrstuvwxyz", 1, - "... with an even lower mem_thresh"); -} - -static void -test_sort_nothing(TestBatchRunner *runner) { - BlobSortEx *sortex = BlobSortEx_new(0x1000000, NULL); - BlobSortEx_Flip(sortex); - TEST_TRUE(runner, BlobSortEx_Fetch(sortex) == NULL, - "Sorting nothing returns undef"); - DECREF(sortex); -} - -static void -test_sort_packed_ints(TestBatchRunner *runner) { - size_t num_ints = 11001; - Vector *blobs = Vec_new(num_ints); - - for (uint32_t i = 0; i < num_ints; ++i) { - uint8_t buf[4]; - buf[0] = (uint8_t)((i >> 24) & 0xFF); - buf[1] = (uint8_t)((i >> 16) & 0xFF); - buf[2] = (uint8_t)((i >> 8) & 0xFF); - buf[3] = (uint8_t)(i & 0xFF); - Blob *blob = Blob_new((char*)buf, 4); - Vec_Push(blobs, (Obj*)blob); - } - - S_test_sort(runner, blobs, 5000, "Sorting packed integers..."); - - DECREF(blobs); -} - -static void -test_sort_random_strings(TestBatchRunner *runner) { - size_t num_strings = 1001; - Vector *blobs = Vec_new(num_strings); - - for (uint32_t i = 0; i < num_strings; ++i) { - uint8_t buf[1201]; - int size = rand() % 1200 + 1; - for (int i = 0; i < size; ++i) { - buf[i] = (uint8_t)(rand() % 256); - } - Blob *blob = Blob_new((char*)buf, (size_t)size); - Vec_Push(blobs, (Obj*)blob); - } - - Vec_Sort(blobs); - S_test_sort(runner, blobs, 15000, - "Random binary strings of random length"); - - DECREF(blobs); -} - -static void -test_run(TestBatchRunner *runner) { - Vector *letters = Vec_new(26); - for (char i = 0; i < 26; ++i) { - char ch = 'a' + i; - Blob *blob = Blob_new(&ch, 1); - Vec_Push(letters, (Obj*)blob); - } - BlobSortEx *run = BlobSortEx_new(0x1000000, letters); - BlobSortEx_Set_Mem_Thresh(run, 5); - - BlobSortEx_Refill(run); - TEST_INT_EQ(runner, BlobSortEx_Buffer_Count(run), 5, - "Refill doesn't exceed memory threshold"); - - Obj *endpost = BlobSortEx_Peek_Last(run); - Blob *wanted = Blob_new("e", 1); - TEST_TRUE(runner, Blob_Equals(wanted, endpost), "Peek_Last"); - - Vector *elems = Vec_new(26); - do { - while (BlobSortEx_Buffer_Count(run) > 0) { - Obj *object = BlobSortEx_Fetch(run); - Vec_Push(elems, object); - } - } while (BlobSortEx_Refill(run) > 0); - TEST_TRUE(runner, Vec_Equals(elems, (Obj*)letters), "retrieve all elems"); - - DECREF(elems); - DECREF(wanted); - DECREF(endpost); - DECREF(letters); - DECREF(run); -} - -void -TestSortExternal_Run_IMP(TestSortExternal *self, TestBatchRunner *runner) { - TestBatchRunner_Plan(runner, (TestBatch*)self, 19); - - srand((unsigned int)time((time_t*)NULL)); - S_init_blobs(); - test_bbsortex(runner); - test_clear_buffer(runner); - test_sort_letters(runner); - test_sort_nothing(runner); - test_sort_packed_ints(runner); - test_sort_random_strings(runner); - test_run(runner); - S_destroy_blobs(); -} - - http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/core/Lucy/Test/Util/TestSortExternal.cfh ---------------------------------------------------------------------- diff --git a/core/Lucy/Test/Util/TestSortExternal.cfh b/core/Lucy/Test/Util/TestSortExternal.cfh deleted file mode 100644 index 028b322..0000000 --- a/core/Lucy/Test/Util/TestSortExternal.cfh +++ /dev/null @@ -1,29 +0,0 @@ -/* Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -parcel TestLucy; - -class Lucy::Test::Util::TestSortExternal - inherits Clownfish::TestHarness::TestBatch { - - inert incremented TestSortExternal* - new(); - - void - Run(TestSortExternal *self, TestBatchRunner *runner); -} - - http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/core/TestLucy.c ---------------------------------------------------------------------- diff --git a/core/TestLucy.c b/core/TestLucy.c deleted file mode 100644 index f499da5..0000000 --- a/core/TestLucy.c +++ /dev/null @@ -1,22 +0,0 @@ -/* Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "testlucy_parcel.h" - -void -testlucy_init_parcel() { -} - http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/core/TestLucy.cfp ---------------------------------------------------------------------- diff --git a/core/TestLucy.cfp b/core/TestLucy.cfp deleted file mode 100644 index a3aef6e..0000000 --- a/core/TestLucy.cfp +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "TestLucy", - "version": "v0.5.0", - "prerequisites": { - "Clownfish": "v0.5.0", - "Lucy": "v0.5.0" - } -} http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/go/build.go ---------------------------------------------------------------------- diff --git a/go/build.go b/go/build.go index 3fea7a0..febcf78 100644 --- a/go/build.go +++ b/go/build.go @@ -118,6 +118,7 @@ func configure() { func runCFC() { hierarchy := cfc.NewHierarchy("autogen") hierarchy.AddSourceDir("../core") + hierarchy.AddSourceDir("../test") hierarchy.Build() autogenHeader := "Auto-generated by build.go.\n" coreBinding := cfc.NewBindCore(hierarchy, autogenHeader, "") http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/perl/Build.PL ---------------------------------------------------------------------- diff --git a/perl/Build.PL b/perl/Build.PL index c37cf3f..3f4ff07 100644 --- a/perl/Build.PL +++ b/perl/Build.PL @@ -28,6 +28,7 @@ my $SNOWSTOP_SRC_DIR = catdir( $MODULES_DIR, qw( analysis snowstop source ) ); my $UCD_INC_DIR = catdir( $MODULES_DIR, qw( unicode ucd ) ); my $UTF8PROC_SRC_DIR = catdir( $MODULES_DIR, qw( unicode utf8proc ) ); my $CORE_SOURCE_DIR = catdir( @BASE_PATH, 'core' ); +my $TEST_SOURCE_DIR = catdir( @BASE_PATH, 'test' ); my $XS_SOURCE_DIR = 'xs'; my @cf_linker_flags = Clownfish::CFC::Perl::Build->cf_linker_flags( @@ -73,15 +74,14 @@ my $builder = Lucy::Build->new( $UTF8PROC_SRC_DIR, ], clownfish_params => { - source => [ - $CORE_SOURCE_DIR, - ], + source => [ $CORE_SOURCE_DIR, $TEST_SOURCE_DIR ], modules => [ { name => 'Lucy', parcels => [ 'Lucy', 'TestLucy' ], c_source_dirs => [ $CORE_SOURCE_DIR, + $TEST_SOURCE_DIR, $XS_SOURCE_DIR, $SNOWSTEM_SRC_DIR, $SNOWSTOP_SRC_DIR, http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test.c ---------------------------------------------------------------------- diff --git a/test/Lucy/Test.c b/test/Lucy/Test.c new file mode 100644 index 0000000..0edda50 --- /dev/null +++ b/test/Lucy/Test.c @@ -0,0 +1,161 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define CFISH_USE_SHORT_NAMES +#define TESTLUCY_USE_SHORT_NAMES + +#include "Lucy/Test.h" + +#include "Clownfish/TestHarness/TestBatch.h" +#include "Clownfish/TestHarness/TestSuite.h" + +#include "Lucy/Test/Analysis/TestAnalyzer.h" +#include "Lucy/Test/Analysis/TestCaseFolder.h" +#include "Lucy/Test/Analysis/TestNormalizer.h" +#include "Lucy/Test/Analysis/TestPolyAnalyzer.h" +#include "Lucy/Test/Analysis/TestRegexTokenizer.h" +#include "Lucy/Test/Analysis/TestSnowballStemmer.h" +#include "Lucy/Test/Analysis/TestSnowballStopFilter.h" +#include "Lucy/Test/Analysis/TestStandardTokenizer.h" +#include "Lucy/Test/Highlight/TestHeatMap.h" +#include "Lucy/Test/Highlight/TestHighlighter.h" +#include "Lucy/Test/Index/TestDocWriter.h" +#include "Lucy/Test/Index/TestHighlightWriter.h" +#include "Lucy/Test/Index/TestIndexManager.h" +#include "Lucy/Test/Index/TestPolyReader.h" +#include "Lucy/Test/Index/TestPostingListWriter.h" +#include "Lucy/Test/Index/TestSegWriter.h" +#include "Lucy/Test/Index/TestSegment.h" +#include "Lucy/Test/Index/TestSnapshot.h" +#include "Lucy/Test/Index/TestSortWriter.h" +#include "Lucy/Test/Index/TestTermInfo.h" +#include "Lucy/Test/Object/TestBitVector.h" +#include "Lucy/Test/Object/TestI32Array.h" +#include "Lucy/Test/Plan/TestBlobType.h" +#include "Lucy/Test/Plan/TestFieldMisc.h" +#include "Lucy/Test/Plan/TestFieldType.h" +#include "Lucy/Test/Plan/TestFullTextType.h" +#include "Lucy/Test/Plan/TestNumericType.h" +#include "Lucy/Test/Search/TestLeafQuery.h" +#include "Lucy/Test/Search/TestMatchAllQuery.h" +#include "Lucy/Test/Search/TestNOTQuery.h" +#include "Lucy/Test/Search/TestNoMatchQuery.h" +#include "Lucy/Test/Search/TestPhraseQuery.h" +#include "Lucy/Test/Search/TestPolyQuery.h" +#include "Lucy/Test/Search/TestQueryParserLogic.h" +#include "Lucy/Test/Search/TestQueryParserSyntax.h" +#include "Lucy/Test/Search/TestRangeQuery.h" +#include "Lucy/Test/Search/TestReqOptQuery.h" +#include "Lucy/Test/Search/TestSeriesMatcher.h" +#include "Lucy/Test/Search/TestSortSpec.h" +#include "Lucy/Test/Search/TestSpan.h" +#include "Lucy/Test/Search/TestTermQuery.h" +#include "Lucy/Test/Store/TestCompoundFileReader.h" +#include "Lucy/Test/Store/TestCompoundFileWriter.h" +#include "Lucy/Test/Store/TestFSDirHandle.h" +#include "Lucy/Test/Store/TestFSFileHandle.h" +#include "Lucy/Test/Store/TestFSFolder.h" +#include "Lucy/Test/Store/TestFileHandle.h" +#include "Lucy/Test/Store/TestFolder.h" +#include "Lucy/Test/Store/TestIOChunks.h" +#include "Lucy/Test/Store/TestIOPrimitives.h" +#include "Lucy/Test/Store/TestInStream.h" +#include "Lucy/Test/Store/TestRAMDirHandle.h" +#include "Lucy/Test/Store/TestRAMFileHandle.h" +#include "Lucy/Test/Store/TestRAMFolder.h" +#include "Lucy/Test/TestSchema.h" +#include "Lucy/Test/TestSimple.h" +#include "Lucy/Test/Util/TestFreezer.h" +#include "Lucy/Test/Util/TestIndexFileNames.h" +#include "Lucy/Test/Util/TestJson.h" +#include "Lucy/Test/Util/TestMemoryPool.h" +#include "Lucy/Test/Util/TestNumberUtils.h" +#include "Lucy/Test/Util/TestPriorityQueue.h" +#include "Lucy/Test/Util/TestSortExternal.h" + +TestSuite* +Test_create_test_suite() { + TestSuite *suite = TestSuite_new(); + + TestSuite_Add_Batch(suite, (TestBatch*)TestPriQ_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestBitVector_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSortExternal_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestMemPool_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestNumUtil_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestIxFileNames_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestJson_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestFreezer_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestI32Arr_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestRAMFH_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestFSFH_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestInStream_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestFH_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestIOPrimitives_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestIOChunks_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestRAMDH_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestFSDH_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestFSFolder_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestRAMFolder_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestFolder_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestIxManager_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestCFWriter_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestCFReader_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestAnalyzer_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestPolyAnalyzer_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestCaseFolder_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestRegexTokenizer_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSnowStop_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSnowStemmer_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestNormalizer_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestStandardTokenizer_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSnapshot_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestTermInfo_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestFieldMisc_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestBatchSchema_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestDocWriter_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestHLWriter_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestPListWriter_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSegWriter_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSortWriter_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestPolyReader_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestFullTextType_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestBlobType_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestNumericType_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestFType_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSeg_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestHighlighter_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSimple_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSpan_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestHeatMap_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestTermQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestPhraseQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSortSpec_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestRangeQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestANDQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestMatchAllQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestNOTQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestReqOptQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestLeafQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestNoMatchQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestSeriesMatcher_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestORQuery_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestQPLogic_new()); + TestSuite_Add_Batch(suite, (TestBatch*)TestQPSyntax_new()); + + return suite; +} + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test.cfh ---------------------------------------------------------------------- diff --git a/test/Lucy/Test.cfh b/test/Lucy/Test.cfh new file mode 100644 index 0000000..3b39130 --- /dev/null +++ b/test/Lucy/Test.cfh @@ -0,0 +1,26 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +parcel TestLucy; + +/** Lucy test suite. + */ +inert class Lucy::Test { + inert incremented TestSuite* + create_test_suite(); +} + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestAnalyzer.c ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestAnalyzer.c b/test/Lucy/Test/Analysis/TestAnalyzer.c new file mode 100644 index 0000000..6df3223 --- /dev/null +++ b/test/Lucy/Test/Analysis/TestAnalyzer.c @@ -0,0 +1,69 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define TESTLUCY_USE_SHORT_NAMES +#include "Lucy/Util/ToolSet.h" + +#include "Clownfish/TestHarness/TestBatchRunner.h" +#include "Lucy/Test.h" +#include "Lucy/Test/TestUtils.h" +#include "Lucy/Test/Analysis/TestAnalyzer.h" +#include "Lucy/Analysis/Analyzer.h" +#include "Lucy/Analysis/Inversion.h" + +TestAnalyzer* +TestAnalyzer_new() { + return (TestAnalyzer*)Class_Make_Obj(TESTANALYZER); +} + +DummyAnalyzer* +DummyAnalyzer_new() { + DummyAnalyzer *self = (DummyAnalyzer*)Class_Make_Obj(DUMMYANALYZER); + return DummyAnalyzer_init(self); +} + +DummyAnalyzer* +DummyAnalyzer_init(DummyAnalyzer *self) { + return (DummyAnalyzer*)Analyzer_init((Analyzer*)self); +} + +Inversion* +DummyAnalyzer_Transform_IMP(DummyAnalyzer *self, Inversion *inversion) { + UNUSED_VAR(self); + return (Inversion*)INCREF(inversion); +} + +static void +test_analysis(TestBatchRunner *runner) { + DummyAnalyzer *analyzer = DummyAnalyzer_new(); + String *source = Str_newf("foo bar baz"); + Vector *wanted = Vec_new(1); + Vec_Push(wanted, (Obj*)Str_newf("foo bar baz")); + TestUtils_test_analyzer(runner, (Analyzer*)analyzer, source, wanted, + "test basic analysis"); + DECREF(wanted); + DECREF(source); + DECREF(analyzer); +} + +void +TestAnalyzer_Run_IMP(TestAnalyzer *self, TestBatchRunner *runner) { + TestBatchRunner_Plan(runner, (TestBatch*)self, 3); + test_analysis(runner); +} + + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestAnalyzer.cfh ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestAnalyzer.cfh b/test/Lucy/Test/Analysis/TestAnalyzer.cfh new file mode 100644 index 0000000..661e391 --- /dev/null +++ b/test/Lucy/Test/Analysis/TestAnalyzer.cfh @@ -0,0 +1,39 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +parcel TestLucy; + +class Lucy::Test::Analysis::TestAnalyzer + inherits Clownfish::TestHarness::TestBatch { + + inert incremented TestAnalyzer* + new(); + + void + Run(TestAnalyzer *self, TestBatchRunner *runner); +} + +class Lucy::Test::Analysis::DummyAnalyzer inherits Lucy::Analysis::Analyzer { + inert incremented DummyAnalyzer* + new(); + + inert DummyAnalyzer* + init(DummyAnalyzer *self); + + public incremented Inversion* + Transform(DummyAnalyzer *self, Inversion *inversion); +} + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestCaseFolder.c ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestCaseFolder.c b/test/Lucy/Test/Analysis/TestCaseFolder.c new file mode 100644 index 0000000..115362d --- /dev/null +++ b/test/Lucy/Test/Analysis/TestCaseFolder.c @@ -0,0 +1,73 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define C_TESTLUCY_TESTCASEFOLDER +#define TESTLUCY_USE_SHORT_NAMES +#include "Lucy/Util/ToolSet.h" + +#include "Clownfish/Boolean.h" +#include "Clownfish/TestHarness/TestBatchRunner.h" +#include "Lucy/Test.h" +#include "Lucy/Test/TestUtils.h" +#include "Lucy/Test/Analysis/TestCaseFolder.h" +#include "Lucy/Analysis/CaseFolder.h" + +TestCaseFolder* +TestCaseFolder_new() { + return (TestCaseFolder*)Class_Make_Obj(TESTCASEFOLDER); +} + +static void +test_Dump_Load_and_Equals(TestBatchRunner *runner) { + CaseFolder *case_folder = CaseFolder_new(); + CaseFolder *other = CaseFolder_new(); + Obj *dump = (Obj*)CaseFolder_Dump(case_folder); + CaseFolder *clone = (CaseFolder*)CaseFolder_Load(other, dump); + + TEST_TRUE(runner, CaseFolder_Equals(case_folder, (Obj*)other), "Equals"); + TEST_FALSE(runner, CaseFolder_Equals(case_folder, (Obj*)CFISH_TRUE), + "Not Equals"); + TEST_TRUE(runner, CaseFolder_Equals(case_folder, (Obj*)clone), + "Dump => Load round trip"); + + DECREF(case_folder); + DECREF(other); + DECREF(dump); + DECREF(clone); +} + +static void +test_analysis(TestBatchRunner *runner) { + CaseFolder *case_folder = CaseFolder_new(); + String *source = Str_newf("caPiTal ofFensE"); + Vector *wanted = Vec_new(1); + Vec_Push(wanted, (Obj*)Str_newf("capital offense")); + TestUtils_test_analyzer(runner, (Analyzer*)case_folder, source, wanted, + "lowercase plain text"); + DECREF(wanted); + DECREF(source); + DECREF(case_folder); +} + +void +TestCaseFolder_Run_IMP(TestCaseFolder *self, TestBatchRunner *runner) { + TestBatchRunner_Plan(runner, (TestBatch*)self, 6); + test_Dump_Load_and_Equals(runner); + test_analysis(runner); +} + + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestCaseFolder.cfh ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestCaseFolder.cfh b/test/Lucy/Test/Analysis/TestCaseFolder.cfh new file mode 100644 index 0000000..6138139 --- /dev/null +++ b/test/Lucy/Test/Analysis/TestCaseFolder.cfh @@ -0,0 +1,29 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +parcel TestLucy; + +class Lucy::Test::Analysis::TestCaseFolder + inherits Clownfish::TestHarness::TestBatch { + + inert incremented TestCaseFolder* + new(); + + void + Run(TestCaseFolder *self, TestBatchRunner *runner); +} + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestNormalizer.c ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestNormalizer.c b/test/Lucy/Test/Analysis/TestNormalizer.c new file mode 100644 index 0000000..6bb0fb7 --- /dev/null +++ b/test/Lucy/Test/Analysis/TestNormalizer.c @@ -0,0 +1,188 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stdlib.h> + +#define C_TESTLUCY_TESTNORMALIZER +#define C_LUCY_NORMALIZER +#define TESTLUCY_USE_SHORT_NAMES +#include "Lucy/Util/ToolSet.h" + +#include "Clownfish/Boolean.h" +#include "Clownfish/TestHarness/TestBatchRunner.h" +#include "Clownfish/TestHarness/TestUtils.h" +#include "Lucy/Test.h" +#include "Lucy/Test/Analysis/TestNormalizer.h" +#include "Lucy/Analysis/Normalizer.h" +#include "Lucy/Store/FSFolder.h" +#include "Lucy/Test/TestUtils.h" +#include "Lucy/Util/Json.h" +#include "utf8proc.h" + +TestNormalizer* +TestNormalizer_new() { + return (TestNormalizer*)Class_Make_Obj(TESTNORMALIZER); +} + +static void +test_Dump_Load_and_Equals(TestBatchRunner *runner) { + Normalizer *normalizer[4]; + + String *NFC = SSTR_WRAP_C("NFC"); + String *NFKC = SSTR_WRAP_C("NFKC"); + + normalizer[0] = Normalizer_new(NFKC, true, false); + normalizer[1] = Normalizer_new(NFC, true, false); + normalizer[2] = Normalizer_new(NFKC, false, false); + normalizer[3] = Normalizer_new(NFKC, true, true); + + TEST_FALSE(runner, + Normalizer_Equals(normalizer[0], (Obj*)normalizer[1]), + "Equals() false with different normalization form"); + TEST_FALSE(runner, + Normalizer_Equals(normalizer[0], (Obj*)normalizer[2]), + "Equals() false with different case_fold flag"); + TEST_FALSE(runner, + Normalizer_Equals(normalizer[0], (Obj*)normalizer[3]), + "Equals() false with different strip_accents flag"); + + for (int i = 0; i < 4; ++i) { + Obj *dump = (Obj*)Normalizer_Dump(normalizer[i]); + Normalizer *clone = (Normalizer*)Normalizer_Load(normalizer[i], dump); + + TEST_TRUE(runner, + Normalizer_Equals(normalizer[i], (Obj*)clone), + "Dump => Load round trip"); + + DECREF(normalizer[i]); + DECREF(dump); + DECREF(clone); + } +} + +static void +test_normalization(TestBatchRunner *runner) { + FSFolder *modules_folder = TestUtils_modules_folder(); + if (modules_folder == NULL) { + SKIP(runner, 13, "Can't locate test data"); + return; + } + + String *path = Str_newf("unicode/utf8proc/tests.json"); + Vector *tests = (Vector*)Json_slurp_json((Folder*)modules_folder, path); + if (!tests) { RETHROW(Err_get_error()); } + + for (size_t i = 0, max = Vec_Get_Size(tests); i < max; i++) { + Hash *test = (Hash*)Vec_Fetch(tests, i); + String *form = (String*)Hash_Fetch_Utf8( + test, "normalization_form", 18); + bool case_fold = Bool_Get_Value((Boolean*)Hash_Fetch_Utf8( + test, "case_fold", 9)); + bool strip_accents = Bool_Get_Value((Boolean*)Hash_Fetch_Utf8( + test, "strip_accents", 13)); + Normalizer *normalizer = Normalizer_new(form, case_fold, strip_accents); + Vector *words = (Vector*)Hash_Fetch_Utf8(test, "words", 5); + Vector *norms = (Vector*)Hash_Fetch_Utf8(test, "norms", 5); + for (size_t j = 0, max = Vec_Get_Size(words); j < max; j++) { + String *word = (String*)Vec_Fetch(words, j); + Vector *got = Normalizer_Split(normalizer, word); + String *norm = (String*)Vec_Fetch(got, 0); + char *fstr = Str_To_Utf8(form); + char *wstr = Str_To_Utf8(word); + TEST_TRUE(runner, + norm + && Str_is_a(norm, STRING) + && Str_Equals(norm, Vec_Fetch(norms, j)), + "Normalize %s %d %d: %s", fstr, + case_fold, strip_accents, wstr + ); + free(fstr); + free(wstr); + DECREF(got); + } + DECREF(normalizer); + } + + DECREF(tests); + DECREF(modules_folder); + DECREF(path); +} + +static void +test_utf8proc_normalization(TestBatchRunner *runner) { + SKIP(runner, 1, + "utf8proc can't handle control chars or Unicode non-chars"); + return; + + for (int32_t i = 0; i < 100; i++) { + String *source = TestUtils_random_string(rand() % 40); + + // Normalize once. + uint8_t *normalized; + int32_t check = utf8proc_map((const uint8_t*)Str_Get_Ptr8(source), + (ssize_t)Str_Get_Size(source), + &normalized, + UTF8PROC_STABLE | + UTF8PROC_COMPOSE | + UTF8PROC_COMPAT | + UTF8PROC_CASEFOLD); + if (check < 0) { + lucy_Json_set_tolerant(1); + String *json = lucy_Json_to_json((Obj*)source); + if (!json) { + json = Str_newf("[failed to encode]"); + } + char *str = Str_To_Utf8(json); + FAIL(runner, "Failed to normalize: %s", str); + free(str); + DECREF(json); + DECREF(source); + return; + } + + // Normalize again. + size_t normalized_len = strlen((char*)normalized); + uint8_t *dupe; + int32_t dupe_check = utf8proc_map(normalized, (ssize_t)normalized_len, &dupe, + UTF8PROC_STABLE | + UTF8PROC_COMPOSE | + UTF8PROC_COMPAT | + UTF8PROC_CASEFOLD); + if (dupe_check < 0) { + THROW(ERR, "Unexpected normalization error: %i32", dupe_check); + } + int comparison = strcmp((char*)normalized, (char*)dupe); + free(dupe); + free(normalized); + DECREF(source); + if (comparison != 0) { + FAIL(runner, "Not fully normalized"); + return; + } + } + PASS(runner, "Normalization successful."); +} + +void +TestNormalizer_Run_IMP(TestNormalizer *self, TestBatchRunner *runner) { + TestBatchRunner_Plan(runner, (TestBatch*)self, 21); + test_Dump_Load_and_Equals(runner); + test_normalization(runner); + test_utf8proc_normalization(runner); +} + + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestNormalizer.cfh ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestNormalizer.cfh b/test/Lucy/Test/Analysis/TestNormalizer.cfh new file mode 100644 index 0000000..0200838 --- /dev/null +++ b/test/Lucy/Test/Analysis/TestNormalizer.cfh @@ -0,0 +1,29 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +parcel TestLucy; + +class Lucy::Test::Analysis::TestNormalizer + inherits Clownfish::TestHarness::TestBatch { + + inert incremented TestNormalizer* + new(); + + void + Run(TestNormalizer *self, TestBatchRunner *runner); +} + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestPolyAnalyzer.c ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestPolyAnalyzer.c b/test/Lucy/Test/Analysis/TestPolyAnalyzer.c new file mode 100644 index 0000000..efb539c --- /dev/null +++ b/test/Lucy/Test/Analysis/TestPolyAnalyzer.c @@ -0,0 +1,179 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define C_TESTLUCY_TESTPOLYANALYZER +#define TESTLUCY_USE_SHORT_NAMES +#include "Lucy/Util/ToolSet.h" + +#include "Clownfish/TestHarness/TestBatchRunner.h" +#include "Lucy/Test.h" +#include "Lucy/Test/TestUtils.h" +#include "Lucy/Test/Analysis/TestPolyAnalyzer.h" +#include "Lucy/Analysis/PolyAnalyzer.h" +#include "Lucy/Analysis/Normalizer.h" +#include "Lucy/Analysis/RegexTokenizer.h" +#include "Lucy/Analysis/SnowballStopFilter.h" +#include "Lucy/Analysis/SnowballStemmer.h" +#include "Lucy/Analysis/StandardTokenizer.h" + +TestPolyAnalyzer* +TestPolyAnalyzer_new() { + return (TestPolyAnalyzer*)Class_Make_Obj(TESTPOLYANALYZER); +} + +static void +test_Dump_Load_and_Equals(TestBatchRunner *runner) { + if (!RegexTokenizer_is_available()) { + SKIP(runner, 3, "RegexTokenizer not available"); + return; + } + + String *EN = SSTR_WRAP_C("en"); + String *ES = SSTR_WRAP_C("es"); + PolyAnalyzer *analyzer = PolyAnalyzer_new(EN, NULL); + PolyAnalyzer *other = PolyAnalyzer_new(ES, NULL); + Obj *dump = (Obj*)PolyAnalyzer_Dump(analyzer); + Obj *other_dump = (Obj*)PolyAnalyzer_Dump(other); + PolyAnalyzer *clone = (PolyAnalyzer*)PolyAnalyzer_Load(other, dump); + PolyAnalyzer *other_clone + = (PolyAnalyzer*)PolyAnalyzer_Load(other, other_dump); + + TEST_FALSE(runner, PolyAnalyzer_Equals(analyzer, (Obj*)other), + "Equals() false with different language"); + TEST_TRUE(runner, PolyAnalyzer_Equals(analyzer, (Obj*)clone), + "Dump => Load round trip"); + TEST_TRUE(runner, PolyAnalyzer_Equals(other, (Obj*)other_clone), + "Dump => Load round trip"); + + DECREF(analyzer); + DECREF(dump); + DECREF(clone); + DECREF(other); + DECREF(other_dump); + DECREF(other_clone); +} + +static void +test_analysis(TestBatchRunner *runner) { + String *EN = SSTR_WRAP_C("en"); + String *source_text = Str_newf("Eats, shoots and leaves."); + Normalizer *normalizer = Normalizer_new(NULL, true, false); + StandardTokenizer *tokenizer = StandardTokenizer_new(); + SnowballStopFilter *stopfilter = SnowStop_new(EN, NULL); + SnowballStemmer *stemmer = SnowStemmer_new(EN); + + { + Vector *analyzers = Vec_new(0); + PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers); + Vector *expected = Vec_new(1); + Vec_Push(expected, INCREF(source_text)); + TestUtils_test_analyzer(runner, (Analyzer*)polyanalyzer, source_text, + expected, "No sub analyzers"); + DECREF(expected); + DECREF(polyanalyzer); + DECREF(analyzers); + } + + { + Vector *analyzers = Vec_new(0); + Vec_Push(analyzers, INCREF(normalizer)); + PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers); + Vector *expected = Vec_new(1); + Vec_Push(expected, (Obj*)Str_newf("eats, shoots and leaves.")); + TestUtils_test_analyzer(runner, (Analyzer*)polyanalyzer, source_text, + expected, "With Normalizer"); + DECREF(expected); + DECREF(polyanalyzer); + DECREF(analyzers); + } + + { + Vector *analyzers = Vec_new(0); + Vec_Push(analyzers, INCREF(normalizer)); + Vec_Push(analyzers, INCREF(tokenizer)); + PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers); + Vector *expected = Vec_new(1); + Vec_Push(expected, (Obj*)Str_newf("eats")); + Vec_Push(expected, (Obj*)Str_newf("shoots")); + Vec_Push(expected, (Obj*)Str_newf("and")); + Vec_Push(expected, (Obj*)Str_newf("leaves")); + TestUtils_test_analyzer(runner, (Analyzer*)polyanalyzer, source_text, + expected, "With StandardTokenizer"); + DECREF(expected); + DECREF(polyanalyzer); + DECREF(analyzers); + } + + { + Vector *analyzers = Vec_new(0); + Vec_Push(analyzers, INCREF(normalizer)); + Vec_Push(analyzers, INCREF(tokenizer)); + Vec_Push(analyzers, INCREF(stopfilter)); + PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers); + Vector *expected = Vec_new(1); + Vec_Push(expected, (Obj*)Str_newf("eats")); + Vec_Push(expected, (Obj*)Str_newf("shoots")); + Vec_Push(expected, (Obj*)Str_newf("leaves")); + TestUtils_test_analyzer(runner, (Analyzer*)polyanalyzer, source_text, + expected, "With SnowballStopFilter"); + DECREF(expected); + DECREF(polyanalyzer); + DECREF(analyzers); + } + + { + Vector *analyzers = Vec_new(0); + Vec_Push(analyzers, INCREF(normalizer)); + Vec_Push(analyzers, INCREF(tokenizer)); + Vec_Push(analyzers, INCREF(stopfilter)); + Vec_Push(analyzers, INCREF(stemmer)); + PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers); + Vector *expected = Vec_new(1); + Vec_Push(expected, (Obj*)Str_newf("eat")); + Vec_Push(expected, (Obj*)Str_newf("shoot")); + Vec_Push(expected, (Obj*)Str_newf("leav")); + TestUtils_test_analyzer(runner, (Analyzer*)polyanalyzer, source_text, + expected, "With SnowballStemmer"); + DECREF(expected); + DECREF(polyanalyzer); + DECREF(analyzers); + } + + DECREF(stemmer); + DECREF(stopfilter); + DECREF(tokenizer); + DECREF(normalizer); + DECREF(source_text); +} + +static void +test_Get_Analyzers(TestBatchRunner *runner) { + Vector *analyzers = Vec_new(0); + PolyAnalyzer *analyzer = PolyAnalyzer_new(NULL, analyzers); + TEST_TRUE(runner, PolyAnalyzer_Get_Analyzers(analyzer) == analyzers, + "Get_Analyzers()"); + DECREF(analyzer); + DECREF(analyzers); +} + +void +TestPolyAnalyzer_Run_IMP(TestPolyAnalyzer *self, TestBatchRunner *runner) { + TestBatchRunner_Plan(runner, (TestBatch*)self, 19); + test_Dump_Load_and_Equals(runner); + test_analysis(runner); + test_Get_Analyzers(runner); +} + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestPolyAnalyzer.cfh ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestPolyAnalyzer.cfh b/test/Lucy/Test/Analysis/TestPolyAnalyzer.cfh new file mode 100644 index 0000000..cd50a76 --- /dev/null +++ b/test/Lucy/Test/Analysis/TestPolyAnalyzer.cfh @@ -0,0 +1,29 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +parcel TestLucy; + +class Lucy::Test::Analysis::TestPolyAnalyzer + inherits Clownfish::TestHarness::TestBatch { + + inert incremented TestPolyAnalyzer* + new(); + + void + Run(TestPolyAnalyzer *self, TestBatchRunner *runner); +} + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestRegexTokenizer.c ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestRegexTokenizer.c b/test/Lucy/Test/Analysis/TestRegexTokenizer.c new file mode 100644 index 0000000..d24ea1d --- /dev/null +++ b/test/Lucy/Test/Analysis/TestRegexTokenizer.c @@ -0,0 +1,75 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define C_TESTLUCY_TESTREGEXTOKENIZER +#define TESTLUCY_USE_SHORT_NAMES +#include "Lucy/Util/ToolSet.h" + +#include "Clownfish/TestHarness/TestBatchRunner.h" +#include "Lucy/Test.h" +#include "Lucy/Test/Analysis/TestRegexTokenizer.h" +#include "Lucy/Analysis/RegexTokenizer.h" + + +TestRegexTokenizer* +TestRegexTokenizer_new() { + return (TestRegexTokenizer*)Class_Make_Obj(TESTREGEXTOKENIZER); +} + +static void +test_Dump_Load_and_Equals(TestBatchRunner *runner) { + if (!RegexTokenizer_is_available()) { + SKIP(runner, 3, "RegexTokenizer not available"); + return; + } + + RegexTokenizer *word_char_tokenizer + = RegexTokenizer_new(SSTR_WRAP_C("\\w+")); + RegexTokenizer *whitespace_tokenizer + = RegexTokenizer_new(SSTR_WRAP_C("\\S+")); + Obj *word_char_dump = RegexTokenizer_Dump(word_char_tokenizer); + Obj *whitespace_dump = RegexTokenizer_Dump(whitespace_tokenizer); + RegexTokenizer *word_char_clone + = RegexTokenizer_Load(whitespace_tokenizer, word_char_dump); + RegexTokenizer *whitespace_clone + = RegexTokenizer_Load(whitespace_tokenizer, whitespace_dump); + + TEST_FALSE(runner, + RegexTokenizer_Equals(word_char_tokenizer, (Obj*)whitespace_tokenizer), + "Equals() false with different pattern"); + TEST_TRUE(runner, + RegexTokenizer_Equals(word_char_tokenizer, (Obj*)word_char_clone), + "Dump => Load round trip"); + TEST_TRUE(runner, + RegexTokenizer_Equals(whitespace_tokenizer, (Obj*)whitespace_clone), + "Dump => Load round trip"); + + DECREF(word_char_tokenizer); + DECREF(word_char_dump); + DECREF(word_char_clone); + DECREF(whitespace_tokenizer); + DECREF(whitespace_dump); + DECREF(whitespace_clone); +} + +void +TestRegexTokenizer_Run_IMP(TestRegexTokenizer *self, TestBatchRunner *runner) { + TestBatchRunner_Plan(runner, (TestBatch*)self, 3); + test_Dump_Load_and_Equals(runner); +} + + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestRegexTokenizer.cfh ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestRegexTokenizer.cfh b/test/Lucy/Test/Analysis/TestRegexTokenizer.cfh new file mode 100644 index 0000000..00e3a60 --- /dev/null +++ b/test/Lucy/Test/Analysis/TestRegexTokenizer.cfh @@ -0,0 +1,29 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +parcel TestLucy; + +class Lucy::Test::Analysis::TestRegexTokenizer + inherits Clownfish::TestHarness::TestBatch { + + inert incremented TestRegexTokenizer* + new(); + + void + Run(TestRegexTokenizer *self, TestBatchRunner *runner); +} + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestSnowballStemmer.c ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestSnowballStemmer.c b/test/Lucy/Test/Analysis/TestSnowballStemmer.c new file mode 100644 index 0000000..4da377f --- /dev/null +++ b/test/Lucy/Test/Analysis/TestSnowballStemmer.c @@ -0,0 +1,116 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define C_TESTLUCY_TESTSNOWBALLSTEMMER +#define TESTLUCY_USE_SHORT_NAMES +#include "Lucy/Util/ToolSet.h" + +#include "Clownfish/HashIterator.h" +#include "Clownfish/TestHarness/TestBatchRunner.h" +#include "Lucy/Test.h" +#include "Lucy/Test/Analysis/TestSnowballStemmer.h" +#include "Lucy/Analysis/SnowballStemmer.h" +#include "Lucy/Store/FSFolder.h" +#include "Lucy/Test/TestUtils.h" +#include "Lucy/Util/Json.h" + +TestSnowballStemmer* +TestSnowStemmer_new() { + return (TestSnowballStemmer*)Class_Make_Obj(TESTSNOWBALLSTEMMER); +} + +static void +test_Dump_Load_and_Equals(TestBatchRunner *runner) { + String *EN = SSTR_WRAP_C("en"); + String *ES = SSTR_WRAP_C("es"); + SnowballStemmer *stemmer = SnowStemmer_new(EN); + SnowballStemmer *other = SnowStemmer_new(ES); + Obj *dump = (Obj*)SnowStemmer_Dump(stemmer); + Obj *other_dump = (Obj*)SnowStemmer_Dump(other); + SnowballStemmer *clone = (SnowballStemmer*)SnowStemmer_Load(other, dump); + SnowballStemmer *other_clone = (SnowballStemmer*)SnowStemmer_Load(other, other_dump); + + TEST_FALSE(runner, + SnowStemmer_Equals(stemmer, (Obj*)other), + "Equals() false with different language"); + TEST_TRUE(runner, + SnowStemmer_Equals(stemmer, (Obj*)clone), + "Dump => Load round trip"); + TEST_TRUE(runner, + SnowStemmer_Equals(other, (Obj*)other_clone), + "Dump => Load round trip"); + + DECREF(stemmer); + DECREF(dump); + DECREF(clone); + DECREF(other); + DECREF(other_dump); + DECREF(other_clone); +} + +static void +test_stemming(TestBatchRunner *runner) { + FSFolder *modules_folder = TestUtils_modules_folder(); + if (modules_folder == NULL) { + SKIP(runner, 150, "Can't locate test data"); + return; + } + + String *path = Str_newf("analysis/snowstem/source/test/tests.json"); + Hash *tests = (Hash*)Json_slurp_json((Folder*)modules_folder, path); + if (!tests) { RETHROW(Err_get_error()); } + + HashIterator *iter = HashIter_new(tests); + while (HashIter_Next(iter)) { + String *iso = HashIter_Get_Key(iter); + char *iso_str = Str_To_Utf8(iso); + Hash *lang_data = (Hash*)HashIter_Get_Value(iter); + Vector *words = (Vector*)Hash_Fetch_Utf8(lang_data, "words", 5); + Vector *stems = (Vector*)Hash_Fetch_Utf8(lang_data, "stems", 5); + SnowballStemmer *stemmer = SnowStemmer_new(iso); + for (size_t i = 0, max = Vec_Get_Size(words); i < max; i++) { + String *word = (String*)Vec_Fetch(words, i); + char *wstr = Str_To_Utf8(word); + Vector *got = SnowStemmer_Split(stemmer, word); + String *stem = (String*)Vec_Fetch(got, 0); + TEST_TRUE(runner, + stem + && Str_is_a(stem, STRING) + && Str_Equals(stem, Vec_Fetch(stems, i)), + "Stem %s: %s", iso_str, wstr + ); + free(wstr); + DECREF(got); + } + free(iso_str); + DECREF(stemmer); + } + DECREF(iter); + + DECREF(tests); + DECREF(modules_folder); + DECREF(path); +} + +void +TestSnowStemmer_Run_IMP(TestSnowballStemmer *self, TestBatchRunner *runner) { + TestBatchRunner_Plan(runner, (TestBatch*)self, 153); + test_Dump_Load_and_Equals(runner); + test_stemming(runner); +} + + + http://git-wip-us.apache.org/repos/asf/lucy/blob/572d3564/test/Lucy/Test/Analysis/TestSnowballStemmer.cfh ---------------------------------------------------------------------- diff --git a/test/Lucy/Test/Analysis/TestSnowballStemmer.cfh b/test/Lucy/Test/Analysis/TestSnowballStemmer.cfh new file mode 100644 index 0000000..7c37248 --- /dev/null +++ b/test/Lucy/Test/Analysis/TestSnowballStemmer.cfh @@ -0,0 +1,29 @@ +/* Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +parcel TestLucy; + +class Lucy::Test::Analysis::TestSnowballStemmer nickname TestSnowStemmer + inherits Clownfish::TestHarness::TestBatch { + + inert incremented TestSnowballStemmer* + new(); + + void + Run(TestSnowballStemmer *self, TestBatchRunner *runner); +} + +
