public override bool IncrementToken()
{
    ClearAttributes();
    position++;
    if (position < tokens.Count)
    {
        var token = tokens[position];
        termAtt.SetTermBuffer(token.Word);
        offsetAtt.SetOffset(token.StartIndex, token.EndIndex);
        typeAtt.Type = "Jieba";
        return true;
    }

    End();
    return false;
}

tokens = segmenter.Tokenize(text, TokenizerMode.Search).ToList();

public override TokenStream TokenStream(string fieldName, TextReader reader)
{
    var seg = new JiebaSegmenter();
    TokenStream result = new JiebaTokenizer(seg, reader);
    // This filter is necessary, because the parser converts the queries to lower case.
    result = new LowerCaseFilter(result);
    result = new StopFilter(true, result, StopWords);
    return result;
}

var analyzer = new JiebaAnalyzer();

using (var writer = new IndexWriter(Directory, analyzer, IndexWriter.MaxFieldLength.UNLIMITED))
{
    // replaces older entry if any
    foreach (var sd in data)
    {
        AddToLuceneIndex(sd, writer);
    }

    analyzer.Close();
}

private static string GetKeyWordsSplitBySpace(string keywords, JiebaTokenizer tokenizer)
{
    var result = new StringBuilder();

    var words = tokenizer.Tokenize(keywords);

    foreach (var word in words)
    {
        if (string.IsNullOrWhiteSpace(word.Word))
        {
            continue;
        }

        result.AppendFormat("{0} ", word.Word);
    }

    return result.ToString().Trim();
}

private static IEnumerable<News> SearchQuery(string searchQuery, string searchField = "")
{
    if (string.IsNullOrEmpty(searchQuery.Replace("*", "").Replace("?", "")))
    {
        return new List<News>();
    }

    using (var searcher = new IndexSearcher(Directory, false))
    {
        var hitsLimit = 1000;
        //var analyzer = new StandardAnalyzer(Version.LUCENE_30);
        var analyzer = GetAnalyzer();

        if (!string.IsNullOrEmpty(searchField))
        {
            var parser = new QueryParser(Version.LUCENE_30, searchField, analyzer);
            var query = ParseQuery(searchQuery, parser);
            var hits = searcher.Search(query, hitsLimit).ScoreDocs;
            var results = MapLuceneToDataList(hits, searcher);

            analyzer.Dispose();
            return results;
        }
        else
        {
            var parser = new MultiFieldQueryParser(Version.LUCENE_30, new[] { "Id", "Title", "Content" }, analyzer);
            var query = ParseQuery(searchQuery, parser);
            var hits = searcher.Search(query, null, hitsLimit, Sort.RELEVANCE).ScoreDocs;
            var results = MapLuceneToDataList(hits, searcher);

            analyzer.Close();
            return results;
        }
    }
}