在lucene中增量更新的問題

Question

我正在創建一個程序，可以索引不同文件夾中的許多文本文件。所以這意味着每個文件夾都有文本文件獲取索引，並且其索引存儲在另一個文件夾中。所以這個文件夾就像我電腦中所有文件的通用索引。我使用lucene來實現這一目標，因爲lucene完全支持增量更新。這是我將它用於索引的源代碼。

public class SimpleFileIndexer { 


public static void main(String[] args) throws Exception { 

    int i=0; 
    while(i<2) { 
    File indexDir = new File("C:/Users/Raden/Documents/myindex"); 
    File dataDir = new File("C:/Users/Raden/Documents/indexthis"); 
    String suffix = "txt"; 

    SimpleFileIndexer indexer = new SimpleFileIndexer(); 

    int numIndex = indexer.index(indexDir, dataDir, suffix); 

    System.out.println("Total files indexed " + numIndex); 
    i++; 
    Thread.sleep(1000); 

    } 
} 


private int index(File indexDir, File dataDir, String suffix) throws Exception { 
    RAMDirectory ramDir = new RAMDirectory();   // 1 
    @SuppressWarnings("deprecation") 
    IndexWriter indexWriter = new IndexWriter(
      ramDir,         // 2 
      new StandardAnalyzer(Version.LUCENE_CURRENT), 
      true, 
      IndexWriter.MaxFieldLength.UNLIMITED); 
    indexWriter.setUseCompoundFile(false); 
    indexDirectory(indexWriter, dataDir, suffix); 
    int numIndexed = indexWriter.maxDoc(); 
    indexWriter.optimize(); 
    indexWriter.close(); 

    Directory.copy(ramDir, FSDirectory.open(indexDir), false); // 3 

    return numIndexed; 
} 


private void indexDirectory(IndexWriter indexWriter, File dataDir, String suffix) throws IOException { 
    File[] files = dataDir.listFiles(); 
    for (int i = 0; i < files.length; i++) { 
     File f = files[i]; 
     if (f.isDirectory()) { 
      indexDirectory(indexWriter, f, suffix); 
     } 
     else { 
      indexFileWithIndexWriter(indexWriter, f, suffix); 
     } 
    } 
} 

private void indexFileWithIndexWriter(IndexWriter indexWriter, File f, String suffix) throws IOException { 
    if (f.isHidden() || f.isDirectory() || !f.canRead() || !f.exists()) { 
     return; 
    } 
    if (suffix!=null && !f.getName().endsWith(suffix)) { 
     return; 
    } 
    System.out.println("Indexing file " + f.getCanonicalPath()); 

    Document doc = new Document(); 
    doc.add(new Field("contents", new FileReader(f)));  
doc.add(new Field("filename", f.getCanonicalPath(), Field.Store.YES, Field.Index.ANALYZED)); 
    indexWriter.addDocument(doc); 
} } 

，這是我使用搜索的Lucene創建的索引

public class SimpleSearcher { 

public static void main(String[] args) throws Exception { 

    File indexDir = new File("C:/Users/Raden/Documents/myindex"); 
    String query = "revolution"; 
    int hits = 100; 

    SimpleSearcher searcher = new SimpleSearcher(); 
    searcher.searchIndex(indexDir, query, hits); 

} 

private void searchIndex(File indexDir, String queryStr, int maxHits) throws Exception { 

    Directory directory = FSDirectory.open(indexDir); 

    IndexSearcher searcher = new IndexSearcher(directory); 
    @SuppressWarnings("deprecation") 
    QueryParser parser = new QueryParser(Version.LUCENE_30, "contents", new StandardAnalyzer(Version.LUCENE_CURRENT)); 
    Query query = parser.parse(queryStr); 

    TopDocs topDocs = searcher.search(query, maxHits); 

    ScoreDoc[] hits = topDocs.scoreDocs; 
    for (int i = 0; i < hits.length; i++) { 
     int docId = hits[i].doc; 
     Document d = searcher.doc(docId); 
     System.out.println(d.get("filename")); 
    } 

    System.out.println("Found " + hits.length); 

} 

} 

我現在遇到的問題是源代碼，我在上面創建的索引程序似乎不能做任何增量更新。我的意思是我可以搜索一個文本文件，但只能搜索我已經編入索引的最後一個文件夾中存在的文件，而我已經編入索引的另一個以前的文件夾似乎在搜索結果中缺失，並且沒有顯示。你能告訴我在我的代碼中出了什麼問題嗎？我只是想在我的源代碼中擁有增量更新功能。所以實質上，我的程序似乎是用新的索引覆蓋現有的索引，而不是合併它。

感謝儘管

Answer 1

Directory.copy()覆蓋目標目錄，你需要使用IndexWriter.addIndexes()新目錄的索引合併到主之一。

您也可以重新打開主索引並直接向其添加文檔。 RAMDirectory不一定比正確調整的緩衝區和合並因子設置更有效（請參閱IndexWriter文檔）。

更新：而不是Directory.copy()你需要打開ramDir閱讀和indexDir編寫和在indexDir作家叫.addIndexes，並將它傳遞的ramDir讀者。或者，您可以使用.addIndexesNoOptimize並直接傳遞ramDir（無需打開閱讀器）並在關閉前優化索引。

但實際上，跳過RAMDir並首先在indexDir上打開一個作者可能更容易。更容易更新更改的文件。

例

private int index(File indexDir, File dataDir, String suffix) throws Exception { 
    RAMDirectory ramDir = new RAMDirectory(); 
    IndexWriter indexWriter = new IndexWriter(ramDir, 
     new StandardAnalyzer(Version.LUCENE_CURRENT), true, 
     IndexWriter.MaxFieldLength.UNLIMITED); 
    indexWriter.setUseCompoundFile(false); 
    indexDirectory(indexWriter, dataDir, suffix); 
    int numIndexed = indexWriter.maxDoc(); 
    indexWriter.optimize(); 
    indexWriter.close(); 


    IndexWriter index = new IndexWriter(FSDirectory.open(indexDir), 
     new StandardAnalyzer(Version.LUCENE_CURRENT), true, 
     IndexWriter.MaxFieldLength.UNLIMITED); 
    index.addIndexesNoOptimize(ramDir); 
    index.optimize(); 
    index.close(); 

    return numIndexed; 
} 

但是，眼前這個是沒關係：

private int index(File indexDir, File dataDir, String suffix) throws Exception { 

    IndexWriter index = new IndexWriter(FSDirectory.open(indexDir), 
     new StandardAnalyzer(Version.LUCENE_CURRENT), true, 
     IndexWriter.MaxFieldLength.UNLIMITED); 

    // tweak the settings for your hardware 
    index.setUseCompoundFile(false); 
    index.setRAMBufferSizeMB(256); 
    index.setMergeFactor(30); 

    indexDirectory(index, dataDir, suffix); 

    index.optimize(); 
    int numIndexed = index.maxDoc(); 
    index.close(); 

    // you'll need to update indexDirectory() to keep track of indexed files 
    return numIndexed; 
}