lucene3.6.1 + IKAnalyzer 创建索引库性能问题！

时光荒羽 2012-08-29 03:51:01

分不够可以再加。
先做一下简要的说明：
我从数据库取出两万条数据，在创建索引库的时候前一万条数据只需要1到3秒左右后面就越来越慢直到内存溢出“java.lang.OutOfMemoryError: Java heap space”

下面就贴出关键的代码



Directory dir = null;

IndexWriter indexWriter = null;

IndexWriterConfig conf = null;



Analyzer analyzer = new IKAnalyzer();

dir = FSDirectory.open(new File(INDEX_PATH));

conf = new IndexWriterConfig(Version.LUCENE_36, analyzer);



LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();

//设置segment添加文档(Document)时的合并频率

mergePolicy.setMergeFactor(20);

//设置segment最大合并文档(Document)数

mergePolicy.setMaxMergeDocs(500);

//启用复合式索引文件格式,合并多个segment

mergePolicy.setUseCompoundFile(true);

conf.setMergePolicy(mergePolicy);

//设置索引的打开模式

conf.setOpenMode(OpenMode.CREATE_OR_APPEND);

//创建索引器

indexWriter = new IndexWriter(dir, conf);



ResultSet set = db.getResultSet("select id,subject,content,posted from news");

List<Document> docs = new ArrayList<Document>();

int i = 0;

//这里set里面有2万条数据

while (set.next())

{

    Document doc = new Document(); 

    doc.add(new Field("id", set.getString("id"), Store.YES, Index.NOT_ANALYZED));

    doc.add(new Field("subject", set.getString("subject"), Store.YES, Index.ANALYZED));

    doc.add(new Field("content", set.getString("content"), Store.YES, Index.ANALYZED));

    doc.add(new Field("posted", DateTools.dateToString(set.getDate("posted"), DateTools.Resolution.DAY),         Store.YES, Index.ANALYZED));

    docs.add(doc);

    if (i++ % 500 == 0) 

    {

        indexWriter.addDocuments(docs);

        System.out.println(docs.size());

        docs.clear();

        System.gc();

    }

    //indexWriter.addDocument(doc, analyzer);

    //indexWriter.addDocument(doc);

}