MongoDB 全文搜索解決方案(lucene+IKAnalyzer)

mongodb 解決 全文搜索是個不小的問題

可以用 正則匹配 但是效率很低 往往到大數據量的搜索的時候就會出現 查詢超時等現象

當然也可以用官方的做法(在mongodb的文檔類型中加字段,存分詞結果,

然后從該字段中匹配) 但是我嘗試了 效率比原先的好像還要低
 

后來我嘗試了 lucene+IKAnalyzer 發現效率有所提升啊

原理:lucene 把大文本的數據 利用分詞器 在新建的索引文件中建立索引

取數據的時候從索引文件中取

取出mongodb 中的數據進行 索引的創建

package sample3;

import java.io.File;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriter.MaxFieldLength;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.wltea.analyzer.lucene.IKAnalyzer;

import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.Mongo;

/**
 * 創建索引
 * @author  zhanghaijun
 *
 */
public class Demo1 {
    public static void main(String[] args) throws Exception {
        //先在數據庫中拿到要創建索引的數據 
        Mongo mongo = new Mongo();
        DB db = mongo.getDB("zhang");
        DBCollection msg = db.getCollection("test3");
        DBCursor cursor = msg.find();
        //是否重新創建索引文件，false：在原有的基礎上追加
        boolean create = true;
        //創建索引
        Directory directory = FSDirectory.open(new File("E:\\lucene\\index"));
        Analyzer analyzer = new IKAnalyzer();//IK中文分詞器 
        IndexWriter indexWriter = new IndexWriter(directory,analyzer,MaxFieldLength.LIMITED);
        boolean exist = cursor.hasNext();
        while(exist){
            //System.out.println(cursor.next().get("text").toString());
            Document doc = new Document();
            Field fieldText = new Field("text",cursor.next().get("text").toString(),Field.Store.YES, 
                      Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
            doc.add(fieldText);
            indexWriter.addDocument(doc);
            exist = cursor.hasNext();
        }
        cursor = null;
        //optimize()方法是對索引進行優化
        indexWriter.optimize();     
        //最后關閉索引
        indexWriter.close();
    }
}

package sample3;

import java.io.File;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.wltea.analyzer.lucene.IKAnalyzer;
import org.wltea.analyzer.lucene.IKQueryParser;
import org.wltea.analyzer.lucene.IKSimilarity;

/**
 * 查找索引
 */
public class Demo2 {
    public static void main(String[] args) throws Exception {
        // onlysearching, so read-only=true
        long starttime = System.currentTimeMillis();
        IndexReader reader =IndexReader.open(FSDirectory.open(new File("E:\\lucene\\index")),true); 
        IndexSearcher searcher = new IndexSearcher(reader);
        searcher.setSimilarity(new IKSimilarity());   //在索引器中使用IKSimilarity相似度評估器 
        //String[] keys = {"4","testtest"};      //關鍵字數組
        //String[] fields = {"id","title"};  //搜索的字段
        //BooleanClause.Occur[] flags = {BooleanClause.Occur.MUST,BooleanClause.Occur.MUST};    //BooleanClause.Occur[]數組,它表示多個條件之間的關系 
        //使用 IKQueryParser類提供的parseMultiField方法構建多字段多條件查詢
        //Query query = IKQueryParser.parseMultiField(fields,keys, flags);     //IKQueryParser多個字段搜索  
        Query query =IKQueryParser.parse("text","上海人");  //IK搜索單個字段       
        IKAnalyzer analyzer = new IKAnalyzer();
        //Query query =MultiFieldQueryParser.parse(Version.LUCENE_CURRENT, keys, fields, flags,analyzer);   //用MultiFieldQueryParser得到query對象   
        System.out.println("query:"+query.toString()); //查詢條件    
        /*TopScoreDocCollector topCollector = TopScoreDocCollector.create(searcher.maxDoc(), false);
        searcher.search(query,topCollector);

        ScoreDoc[] docs = topCollector.topDocs(3).scoreDocs;
        System.out.println(docs.length);*/

        /**
        *得到TopDocs對象之后,可以獲取它的成員變量totalHits和scoreDocs.這兩個成員變量的訪問權限是public的,所以可以直接訪問
        */
        TopDocs topDocs = searcher.search(query,1000001);
        Integer count = topDocs.totalHits;
        ScoreDoc[] scoreDocs = topDocs.scoreDocs;
        for(int i = 0;i<count;i++){
            ScoreDoc scoreDoc = scoreDocs[i];
            Document document = searcher.doc(scoreDoc.doc);
            document.get("text");
        }
        System.out.println("查找數據量:"+count);
        long endtime = System.currentTimeMillis();
        System.out.println(endtime-starttime);
        reader.close(); //關閉索引   
    }
}

package sample3;

import java.net.UnknownHostException;

import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.Mongo;
import com.mongodb.MongoException;

public class Demo3 {
    public static void main(String[] args) throws Exception{
        Mongo mongo = new Mongo();
        DB db = mongo.getDB("zhang");
        DBCollection dbc = db.getCollection("test3");
        DBObject basicdb = new BasicDBObject();
        basicdb.put("$regex","上海人");
        basicdb.put("$options","");
        long startTime = System.currentTimeMillis();
        DBCursor cursor = dbc.find(new BasicDBObject("text",basicdb));
        int j =0;
        while(cursor.hasNext()){
            cursor.next().get("text");
            j++;
        }
        System.out.println("查找數據量"+j);
        long endTime = System.currentTimeMillis();
        System.out.println("未優化前:"+(endTime-startTime));
    }
}

       從索引中找數據  

優化后的:
query:text:上海人 text:上海
查找數據量:1000001
查找用時:2917

      直接從mongodb 正則匹配的

查找數據量1000000
未優化前:6573

還有更好的解決方案嗎 求指導哦