http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.csproj ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.csproj b/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.csproj new file mode 100644 index 0000000..e2cbbb8 --- /dev/null +++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.csproj @@ -0,0 +1,106 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="14.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003";> + <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> + <PropertyGroup> + <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> + <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> + <ProjectGuid>{34A2BCE8-1351-43BD-A365-F50E7C0B2C49}</ProjectGuid> + <OutputType>Library</OutputType> + <AppDesignerFolder>Properties</AppDesignerFolder> + <RootNamespace>Lucene.Net.Analysis.Ja</RootNamespace> + <AssemblyName>Lucene.Net.Tests.Analysis.Kuromoji</AssemblyName> + <TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion> + <FileAlignment>512</FileAlignment> + </PropertyGroup> + <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> + <DebugSymbols>true</DebugSymbols> + <DebugType>full</DebugType> + <Optimize>false</Optimize> + <OutputPath>bin\Debug\</OutputPath> + <DefineConstants>DEBUG;TRACE</DefineConstants> + <ErrorReport>prompt</ErrorReport> + <WarningLevel>4</WarningLevel> + </PropertyGroup> + <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' "> + <DebugType>pdbonly</DebugType> + <Optimize>true</Optimize> + <OutputPath>bin\Release\</OutputPath> + <DefineConstants>TRACE</DefineConstants> + <ErrorReport>prompt</ErrorReport> + <WarningLevel>4</WarningLevel> + </PropertyGroup> + <ItemGroup> + <Reference Include="System" /> + <Reference Include="System.Core" /> + <Reference Include="System.Xml.Linq" /> + <Reference Include="System.Data.DataSetExtensions" /> + <Reference Include="Microsoft.CSharp" /> + <Reference Include="System.Data" /> + <Reference Include="System.Net.Http" /> + <Reference Include="System.Xml" /> + </ItemGroup> + <ItemGroup> + <Compile Include="Dict\TestTokenInfoDictionary.cs" /> + <Compile Include="Dict\UserDictionaryTest.cs" /> + <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="StringMockResourceLoader.cs" /> + <Compile Include="Support\TestApiConsistency.cs" /> + <Compile Include="Support\TestExceptionSerialization.cs" /> + <Compile Include="TestExtendedMode.cs" /> + <Compile Include="TestJapaneseAnalyzer.cs" /> + <Compile Include="TestJapaneseBaseFormFilter.cs" /> + <Compile Include="TestJapaneseBaseFormFilterFactory.cs" /> + <Compile Include="TestJapaneseIterationMarkCharFilter.cs" /> + <Compile Include="TestJapaneseIterationMarkCharFilterFactory.cs" /> + <Compile Include="TestJapaneseKatakanaStemFilter.cs" /> + <Compile Include="TestJapaneseKatakanaStemFilterFactory.cs" /> + <Compile Include="TestJapanesePartOfSpeechStopFilterFactory.cs" /> + <Compile Include="TestJapaneseReadingFormFilter.cs" /> + <Compile Include="TestJapaneseReadingFormFilterFactory.cs" /> + <Compile Include="TestJapaneseTokenizer.cs" /> + <Compile Include="TestJapaneseTokenizerFactory.cs" /> + <Compile Include="TestSearchMode.cs" /> + <Compile Include="Tools\UnknownDictionaryTest.cs" /> + <Compile Include="Util\TestToStringUtil.cs" /> + <Compile Include="..\CommonAssemblyInfo.cs"> + <Link>Properties\CommonAssemblyInfo.cs</Link> + </Compile> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="..\Lucene.Net.Analysis.Common\Lucene.Net.Analysis.Common.csproj"> + <Project>{4ADD0BBC-B900-4715-9526-D871DE8EEA64}</Project> + <Name>Lucene.Net.Analysis.Common</Name> + </ProjectReference> + <ProjectReference Include="..\Lucene.Net.Analysis.Kuromoji\Lucene.Net.Analysis.Kuromoji.csproj"> + <Project>{8408625A-2508-46D5-8519-045183C43724}</Project> + <Name>Lucene.Net.Analysis.Kuromoji</Name> + </ProjectReference> + <ProjectReference Include="..\Lucene.Net.TestFramework\Lucene.Net.TestFramework.csproj"> + <Project>{B2C0D749-CE34-4F62-A15E-00CB2FF5DDB3}</Project> + <Name>Lucene.Net.TestFramework</Name> + </ProjectReference> + <ProjectReference Include="..\Lucene.Net\Lucene.Net.csproj"> + <Project>{5D4AD9BE-1FFB-41AB-9943-25737971BF57}</Project> + <Name>Lucene.Net</Name> + </ProjectReference> + </ItemGroup> + <ItemGroup> + <EmbeddedResource Include="bocchan.utf-8" /> + <None Include="Lucene.Net.Tests.Analysis.Kuromoji.project.json" /> + </ItemGroup> + <ItemGroup> + <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" /> + </ItemGroup> + <ItemGroup> + <EmbeddedResource Include="search-segmentation-tests.txt" /> + <EmbeddedResource Include="userdict.txt" /> + </ItemGroup> + <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> + <!-- To modify your build process, add your task inside one of the targets below and uncomment it. + Other similar extension points exist, see Microsoft.Common.targets. + <Target Name="BeforeBuild"> + </Target> + <Target Name="AfterBuild"> + </Target> + --> +</Project> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.project.json ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.project.json b/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.project.json new file mode 100644 index 0000000..8c631ab --- /dev/null +++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.project.json @@ -0,0 +1,11 @@ +{ + "runtimes": { + "win": {} + }, + "dependencies": { + "NUnit": "3.5.0" + }, + "frameworks": { + "net451": {} + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.xproj ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.xproj b/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.xproj new file mode 100644 index 0000000..f35a3f2 --- /dev/null +++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/Lucene.Net.Tests.Analysis.Kuromoji.xproj @@ -0,0 +1,41 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<Project ToolsVersion="14.0.25420" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003";> + <PropertyGroup> + <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">14.0.25420</VisualStudioVersion> + <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath> + </PropertyGroup> + <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.Props" Condition="'$(VSToolsPath)' != ''" /> + <PropertyGroup Label="Globals"> + <ProjectGuid>f82f0f31-09e7-48fb-b5ff-f3a84627a307</ProjectGuid> + <RootNamespace>Lucene.Net.Analysis.Ja</RootNamespace> + <BaseIntermediateOutputPath Condition="'$(BaseIntermediateOutputPath)'=='' ">.\obj</BaseIntermediateOutputPath> + <OutputPath Condition="'$(OutputPath)'=='' ">.\bin\</OutputPath> + </PropertyGroup> + <PropertyGroup> + <SchemaVersion>2.0</SchemaVersion> + </PropertyGroup> + <ItemGroup> + <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" /> + </ItemGroup> + <Import Project="$(VSToolsPath)\DotNet\Microsoft.DotNet.targets" Condition="'$(VSToolsPath)' != ''" /> +</Project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/Properties/AssemblyInfo.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/Properties/AssemblyInfo.cs b/src/Lucene.Net.Tests.Analysis.Kuromoji/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..4c466cf --- /dev/null +++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/Properties/AssemblyInfo.cs @@ -0,0 +1,38 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("Lucene.Net.Tests.Analysis.Kuromoji")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("34a2bce8-1351-43bd-a365-f50e7c0b2c49")] + +// NOTE: Version information is in CommonAssemblyInfo.cs http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/StringMockResourceLoader.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/StringMockResourceLoader.cs b/src/Lucene.Net.Tests.Analysis.Kuromoji/StringMockResourceLoader.cs new file mode 100644 index 0000000..49819cd --- /dev/null +++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/StringMockResourceLoader.cs @@ -0,0 +1,67 @@ +using Lucene.Net.Analysis.Util; +using System; +using System.IO; +using System.Text; + +namespace Lucene.Net.Analysis.Ja +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + /// <summary>Fake resource loader for tests: works if you want to fake reading a single file</summary> + internal class StringMockResourceLoader : IResourceLoader + { + String text; + + public StringMockResourceLoader(String text) + { + this.text = text; + } + + public virtual Type FindType(String cname) + { + try + { + //return Class.forName(cname).asSubclass(expectedType); + return Type.GetType(cname); + } + catch (Exception e) + { + throw new Exception("Cannot load class: " + cname, e); + } + } + + public virtual T NewInstance<T>(String cname) + { + Type clazz = FindType(cname); + try + { + //return clazz.NewInstance(); + return (T)Activator.CreateInstance(clazz); + } + catch (Exception e) + { + throw new Exception("Cannot create instance: " + cname, e); + } + } + + public virtual Stream OpenResource(String resource) + { + return new MemoryStream(Encoding.UTF8.GetBytes(text)); + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/Support/TestApiConsistency.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/Support/TestApiConsistency.cs b/src/Lucene.Net.Tests.Analysis.Kuromoji/Support/TestApiConsistency.cs new file mode 100644 index 0000000..de33cf3 --- /dev/null +++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/Support/TestApiConsistency.cs @@ -0,0 +1,150 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ + +using Lucene.Net.Attributes; +using Lucene.Net.Support; +using NUnit.Framework; +using System; + +namespace Lucene.Net.Analysis.Ja.Support +{ + /// <summary> + /// LUCENENET specific tests for ensuring API conventions are followed + /// </summary> + public class TestApiConsistency : ApiScanTestBase + { + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestProtectedFieldNames(Type typeFromTargetAssembly) + { + base.TestProtectedFieldNames(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestPrivateFieldNames(Type typeFromTargetAssembly) + { + base.TestPrivateFieldNames(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestPublicFields(Type typeFromTargetAssembly) + { + base.TestPublicFields(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestMethodParameterNames(Type typeFromTargetAssembly) + { + base.TestMethodParameterNames(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestInterfaceNames(Type typeFromTargetAssembly) + { + base.TestInterfaceNames(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestClassNames(Type typeFromTargetAssembly) + { + base.TestClassNames(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestForPropertiesWithNoGetter(Type typeFromTargetAssembly) + { + base.TestForPropertiesWithNoGetter(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestForPropertiesThatReturnArray(Type typeFromTargetAssembly) + { + base.TestForPropertiesThatReturnArray(typeFromTargetAssembly); + } + +#if !NETSTANDARD + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestForMethodsThatReturnWritableArray(Type typeFromTargetAssembly) + { + base.TestForMethodsThatReturnWritableArray(typeFromTargetAssembly); + } +#endif + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestForPublicMembersContainingComparer(Type typeFromTargetAssembly) + { + base.TestForPublicMembersContainingComparer(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestForPublicMembersNamedSize(Type typeFromTargetAssembly) + { + base.TestForPublicMembersNamedSize(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestForPublicMembersContainingNonNetNumeric(Type typeFromTargetAssembly) + { + base.TestForPublicMembersContainingNonNetNumeric(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestForTypesContainingNonNetNumeric(Type typeFromTargetAssembly) + { + base.TestForTypesContainingNonNetNumeric(typeFromTargetAssembly); + } + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestForPublicMembersWithNullableEnum(Type typeFromTargetAssembly) + { + base.TestForPublicMembersWithNullableEnum(typeFromTargetAssembly); + } + + // LUCENENET NOTE: This test is only for identifying members who were changed from + // ICollection, IList or ISet to IEnumerable during the port (that should be changed back) + //[Test, LuceneNetSpecific] + //[TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + //public override void TestForMembersAcceptingOrReturningIEnumerable(Type typeFromTargetAssembly) + //{ + // base.TestForMembersAcceptingOrReturningIEnumerable(typeFromTargetAssembly); + //} + + [Test, LuceneNetSpecific] + [TestCase(typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter))] + public override void TestForMembersAcceptingOrReturningListOrDictionary(Type typeFromTargetAssembly) + { + base.TestForMembersAcceptingOrReturningListOrDictionary(typeFromTargetAssembly); + } + } +} http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/Support/TestExceptionSerialization.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/Support/TestExceptionSerialization.cs b/src/Lucene.Net.Tests.Analysis.Kuromoji/Support/TestExceptionSerialization.cs new file mode 100644 index 0000000..7dcea20 --- /dev/null +++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/Support/TestExceptionSerialization.cs @@ -0,0 +1,54 @@ +#if FEATURE_SERIALIZABLE +using Lucene.Net.Attributes; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Lucene.Net.Support +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + [TestFixture] + public class TestExceptionSerialization : ExceptionSerializationTestBase + { + public static IEnumerable<object> ExceptionTestData + { + get + { + var exceptionTypes = typeof(Lucene.Net.Analysis.Ja.GraphvizFormatter).Assembly.GetTypes().Where(t => typeof(Exception).IsAssignableFrom(t)).Cast<object>(); + + // If the assembly has no exceptions, just provide Exception so the test will pass + if (!exceptionTypes.Any()) + { + return new Type[] { typeof(Exception) }; + } + + return exceptionTypes; + } + } + + [Test, LuceneNetSpecific] + public void AllExceptionsInLuceneNamespaceCanSerialize([ValueSource("ExceptionTestData")]Type luceneException) + { + var instance = TryInstantiate(luceneException); + Assert.That(TypeCanSerialize(instance), string.Format("Unable to serialize {0}", luceneException.FullName)); + } + } +} +#endif \ No newline at end of file http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0f092010/src/Lucene.Net.Tests.Analysis.Kuromoji/TestExtendedMode.cs ---------------------------------------------------------------------- diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/TestExtendedMode.cs b/src/Lucene.Net.Tests.Analysis.Kuromoji/TestExtendedMode.cs new file mode 100644 index 0000000..f4db3d9 --- /dev/null +++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/TestExtendedMode.cs @@ -0,0 +1,82 @@ +using Lucene.Net.Analysis.TokenAttributes; +using Lucene.Net.Util; +using NUnit.Framework; +using System; + +namespace Lucene.Net.Analysis.Ja +{ + /* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + public class TestExtendedMode : BaseTokenStreamTestCase + { + private readonly Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => + { + Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizerMode.EXTENDED); + return new TokenStreamComponents(tokenizer, tokenizer); + }); + + /** simple test for supplementary characters */ + [Test] + public void TestSurrogates() + { + AssertAnalyzesTo(analyzer, "𩬠è±éä¹æ¯ç", + new String[] { "𩬠", "è±", "é", "ä¹", "æ¯", "ç" }); + } + + /** random test ensuring we don't ever split supplementaries */ + [Test] + public void TestSurrogates2() + { + int numIterations = AtLeast(1000); + for (int i = 0; i < numIterations; i++) + { + String s = TestUtil.RandomUnicodeString(Random(), 100); + TokenStream ts = analyzer.GetTokenStream("foo", s); + try + { + ICharTermAttribute termAtt = ts.AddAttribute<ICharTermAttribute>(); + ts.Reset(); + while (ts.IncrementToken()) + { + assertTrue(UnicodeUtil.ValidUTF16String(termAtt)); + } + ts.End(); + } + finally + { + IOUtils.DisposeWhileHandlingException(ts); + } + } + } + + /** blast some random strings through the analyzer */ + [Test] + public void TestRandomStrings() + { + Random random = Random(); + CheckRandomData(random, analyzer, 1000 * RANDOM_MULTIPLIER); + } + + /** blast some random large strings through the analyzer */ + public void TestRandomHugeStrings() + { + Random random = Random(); + CheckRandomData(random, analyzer, 100 * RANDOM_MULTIPLIER, 8192); + } + } +}
