package org.apache.lucene.facet.search;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.junit.Before;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.facet.FacetTestUtils;
import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.TestMultiCLExample;
import org.apache.lucene.facet.example.multiCL.MultiCLIndexer;
import org.apache.lucene.facet.example.multiCL.MultiCLSearcher;
import org.apache.lucene.facet.index.CategoryDocumentBuilder;
import org.apache.lucene.facet.index.params.DefaultFacetIndexingParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.TotalFacetCounts.CreationType;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.SlowRAMDirectory;
import org.apache.lucene.util._TestUtil;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
public class TestTotalFacetCountsCache extends LuceneTestCase {
static final TotalFacetCountsCache TFC = TotalFacetCountsCache.getSingleton();
/**
* Thread class to be used in tests for this method. This thread gets a TFC
* and records times.
*/
private static class TFCThread extends Thread {
private final IndexReader r;
private final DirectoryTaxonomyReader tr;
private final FacetIndexingParams iParams;
TotalFacetCounts tfc;
public TFCThread(IndexReader r, DirectoryTaxonomyReader tr, FacetIndexingParams iParams) {
this.r = r;
this.tr = tr;
this.iParams = iParams;
}
@Override
public void run() {
try {
tfc = TFC.getTotalCounts(r, tr, iParams, null);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
/** Utility method to add a document and facets to an index/taxonomy. */
static void addFacets(FacetIndexingParams iParams, IndexWriter iw,
TaxonomyWriter tw, String... strings) throws IOException {
ArrayList<CategoryPath> cps = new ArrayList<CategoryPath>();
cps.add(new CategoryPath(strings));
CategoryDocumentBuilder builder = new CategoryDocumentBuilder(tw, iParams);
iw.addDocument(builder.setCategoryPaths(cps).build(new Document()));
}
/** Clears the cache and sets its size to one. */
static void initCache() {
TFC.clear();
TFC.setCacheSize(1); // Set to keep one in memory
}
@Override
@Before
public void setUp() throws Exception {
super.setUp();
initCache();
}
/** runs a few instances of {@link MultiCLSearcher} in parallel */
public void testGeneralSynchronization() throws Exception {
int numIters = atLeast(2);
for (int i = 0; i < numIters; i++) {
doTestGeneralSynchronization(_TestUtil.nextInt(random, 2, 4),
random.nextBoolean() ? -1 : _TestUtil.nextInt(random, 1, 10),
_TestUtil.nextInt(random, 0, 3));
}
}
/**
* Run many instances of {@link MultiCLSearcher} in parallel, results should
* be sane. Each instance has a random delay for reading bytes, to ensure
* that threads finish in different order than started.
*/
@Test @Nightly
public void testGeneralSynchronizationBig() throws Exception {
int[] numThreads = new int[] { 2, 3, 5, 8 };
int[] sleepMillis = new int[] { -1, 1, 20, 33 };
int[] cacheSize = new int[] { 0,1,2,3,5 };
for (int size : cacheSize) {
for (int sleep : sleepMillis) {
for (int nThreads : numThreads) {
doTestGeneralSynchronization(nThreads, sleep, size);
}
}
}
}
private void doTestGeneralSynchronization(int numThreads, int sleepMillis,
int cacheSize) throws Exception, CorruptIndexException, IOException,
InterruptedException {
TFC.setCacheSize(cacheSize);
SlowRAMDirectory slowIndexDir = new SlowRAMDirectory(-1, random);
MockDirectoryWrapper indexDir = new MockDirectoryWrapper(random, slowIndexDir);
SlowRAMDirectory slowTaxoDir = new SlowRAMDirectory(-1, random);
MockDirectoryWrapper taxoDir = new MockDirectoryWrapper(random, slowTaxoDir);
// Index documents without the "slowness"
MultiCLIndexer.index(indexDir, taxoDir);
slowIndexDir.setSleepMillis(sleepMillis);
slowTaxoDir.setSleepMillis(sleepMillis);
// Open the slow readers
IndexReader slowIndexReader = IndexReader.open(indexDir);
TaxonomyReader slowTaxoReader = new DirectoryTaxonomyReader(taxoDir);
// Class to perform search and return results as threads
class Multi extends Thread {
private List<FacetResult> results;
private FacetIndexingParams iParams;
private IndexReader indexReader;
private TaxonomyReader taxoReader;
public Multi(IndexReader indexReader, TaxonomyReader taxoReader,
FacetIndexingParams iParams) {
this.indexReader = indexReader;
this.taxoReader = taxoReader;
this.iParams = iParams;
}
public ExampleResult getResults() {
ExampleResult exampleRes = new ExampleResult();
exampleRes.setFacetResults(results);
return exampleRes;
}
@Override
public void run() {
try {
results = MultiCLSearcher.searchWithFacets(indexReader, taxoReader, iParams);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
// Instantiate threads, but do not start them
Multi[] multis = new Multi[numThreads];
for (int i = 0; i < numThreads - 1; i++) {
multis[i] = new Multi(slowIndexReader, slowTaxoReader, MultiCLIndexer.MULTI_IPARAMS);
}
// The last thread uses ONLY the DefaultFacetIndexingParams so that
// it references a different TFC cache. This will still result
// in valid results, but will only search one of the category lists
// instead of all of them.
multis[numThreads - 1] = new Multi(slowIndexReader, slowTaxoReader, new DefaultFacetIndexingParams());
// Gentleman, start your engines
for (Multi m : multis) {
m.start();
}
// Wait for threads and get results
ExampleResult[] multiResults = new ExampleResult[numThreads];
for (int i = 0; i < numThreads; i++) {
multis[i].join();
multiResults[i] = multis[i].getResults();
}
// Each of the (numThreads-1) should have the same predictable
// results, which we test for here.
for (int i = 0; i < numThreads - 1; i++) {
ExampleResult eResults = multiResults[i];
TestMultiCLExample.assertCorrectMultiResults(eResults);
}
// The last thread, which only searched over the
// DefaultFacetIndexingParams,
// has its own results
ExampleResult eResults = multiResults[numThreads - 1];
List<FacetResult> results = eResults.getFacetResults();
assertEquals(3, results.size());
String[] expLabels = new String[] { "5", "5/5", "6/2" };
double[] expValues = new double[] { 0.0, 0.0, 1.0 };
for (int i = 0; i < 3; i++) {
FacetResult result = results.get(i);
assertNotNull("Result should not be null", result);
FacetResultNode resNode = result.getFacetResultNode();
assertEquals("Invalid label", expLabels[i], resNode.getLabel().toString());
assertEquals("Invalid value", expValues[i], resNode.getValue(), 0.0);
assertEquals("Invalid number of subresults", 0, resNode.getNumSubResults());
}
// we're done, close the index reader and the taxonomy.
slowIndexReader.close();
slowTaxoReader.close();
indexDir.close();
taxoDir.close();
}
/**
* Simple test to make sure the TotalFacetCountsManager updates the
* TotalFacetCounts array only when it is supposed to, and whether it
* is recomputed or read from disk.
*/
@Test
public void testGenerationalConsistency() throws Exception {
// Create temporary RAMDirectories
Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
// Create our index/taxonomy writers
IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
DefaultFacetIndexingParams iParams = new DefaultFacetIndexingParams();
// Add a facet to the index
addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
// Commit Changes
writers[0].indexWriter.commit();
writers[0].taxWriter.commit();
// Open readers
IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
// As this is the first time we have invoked the TotalFacetCountsManager,
// we should expect to compute and not read from disk.
TotalFacetCounts totalCounts =
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
int prevGen = assertRecomputed(totalCounts, 0, "after first attempt to get it!");
// Repeating same operation should pull from the cache - not recomputed.
assertTrue("Should be obtained from cache at 2nd attempt",totalCounts ==
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null));
// Repeat the same operation as above. but clear first - now should recompute again
initCache();
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 3rd attempt to get it!");
//store to file
File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
initCache();
TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams, null);
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 4th attempt to get it!");
//clear and load
initCache();
TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
prevGen = assertReadFromDisc(totalCounts, prevGen, "after 5th attempt to get it!");
// Add a new facet to the index, commit and refresh readers
addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
writers[0].indexWriter.close();
writers[0].taxWriter.close();
readers[0].taxReader.refresh();
IndexReader r2 = readers[0].indexReader.reopen();
// Hold on to the 'original' reader so we can do some checks with it
IndexReader origReader = null;
assertTrue("Reader must be updated!", readers[0].indexReader != r2);
// Set the 'original' reader
origReader = readers[0].indexReader;
// Set the new master index Reader
readers[0].indexReader = r2;
// Try to get total-counts the originalReader AGAIN, just for sanity. Should pull from the cache - not recomputed.
assertTrue("Should be obtained from cache at 6th attempt",totalCounts ==
TFC.getTotalCounts(origReader, readers[0].taxReader, iParams, null));
// now use the new reader - should recompute
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
prevGen = assertRecomputed(totalCounts, prevGen, "after updating the index - 7th attempt!");
// try again - should not recompute
assertTrue("Should be obtained from cache at 8th attempt",totalCounts ==
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null));
// delete a doc from the reader and commit - should recompute
origReader.close();
origReader = readers[0].indexReader;
readers[0].indexReader = IndexReader.open(origReader.directory(),false);
initCache();
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
prevGen = assertRecomputed(totalCounts, prevGen, "after opening a writable reader - 9th attempt!");
// now do the delete
readers[0].indexReader.deleteDocument(1);
readers[0].indexReader.commit(null);
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
prevGen = assertRecomputed(totalCounts, prevGen, "after deleting docs the index - 10th attempt!");
origReader.close();
readers[0].close();
r2.close();
outputFile.delete();
IOUtils.close(dirs[0]);
}
private int assertReadFromDisc(TotalFacetCounts totalCounts, int prevGen, String errMsg) {
assertEquals("should read from disk "+errMsg, CreationType.Loaded, totalCounts.createType4test);
int gen4test = totalCounts.gen4test;
assertTrue("should read from disk "+errMsg, gen4test > prevGen);
return gen4test;
}
private int assertRecomputed(TotalFacetCounts totalCounts, int prevGen, String errMsg) {
assertEquals("should recompute "+errMsg, CreationType.Computed, totalCounts.createType4test);
int gen4test = totalCounts.gen4test;
assertTrue("should recompute "+errMsg, gen4test > prevGen);
return gen4test;
}
/**
* This test is to address a bug in a previous version. If a TFC cache is
* written to disk, and then the taxonomy grows (but the index does not change),
* and then the TFC cache is re-read from disk, there will be an exception
* thrown, as the integers are read off of the disk according to taxonomy
* size, which has changed.
*/
@Test
public void testGrowingTaxonomy() throws Exception {
// Create temporary RAMDirectories
Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
// Create our index/taxonomy writers
IndexTaxonomyWriterPair[] writers = FacetTestUtils
.createIndexTaxonomyWriterPair(dirs);
DefaultFacetIndexingParams iParams = new DefaultFacetIndexingParams() {
@Override
protected int fixedPartitionSize() {
return 2;
}
};
// Add a facet to the index
addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
// Commit Changes
writers[0].indexWriter.commit();
writers[0].taxWriter.commit();
IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
// Create TFC and write cache to disk
File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams, null);
// Make the taxonomy grow without touching the index
for (int i = 0; i < 10; i++) {
writers[0].taxWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
}
writers[0].taxWriter.commit();
readers[0].taxReader.refresh();
initCache();
// With the bug, this next call should result in an exception
TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
TotalFacetCounts totalCounts = TFC.getTotalCounts(
readers[0].indexReader, readers[0].taxReader, iParams, null);
assertReadFromDisc(totalCounts, 0, "after reading from disk.");
outputFile.delete();
writers[0].close();
readers[0].close();
IOUtils.close(dirs[0]);
}
/**
* Test that a new TFC is only calculated and placed in memory (by two
* threads who want it at the same time) only once.
*/
@Test
public void testMemoryCacheSynchronization() throws Exception {
SlowRAMDirectory indexDir = new SlowRAMDirectory(-1, null);
SlowRAMDirectory taxoDir = new SlowRAMDirectory(-1, null);
// Write index using 'normal' directories
IndexWriter w = new IndexWriter(indexDir, new IndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
DefaultFacetIndexingParams iParams = new DefaultFacetIndexingParams();
// Add documents and facets
for (int i = 0; i < 1000; i++) {
addFacets(iParams, w, tw, "facet", Integer.toString(i));
}
w.close();
tw.close();
indexDir.setSleepMillis(1);
taxoDir.setSleepMillis(1);
IndexReader r = IndexReader.open(indexDir);
DirectoryTaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// Create and start threads. Thread1 should lock the cache and calculate
// the TFC array. The second thread should block until the first is
// done, then successfully retrieve from the cache without recalculating
// or reading from disk.
TFCThread tfcCalc1 = new TFCThread(r, tr, iParams);
TFCThread tfcCalc2 = new TFCThread(r, tr, iParams);
tfcCalc1.start();
// Give thread 1 a head start to ensure correct sequencing for testing
Thread.sleep(5);
tfcCalc2.start();
tfcCalc1.join();
tfcCalc2.join();
// Since this test ends up with references to the same TFC object, we
// can only test the times to make sure that they are the same.
assertRecomputed(tfcCalc1.tfc, 0, "thread 1 should recompute");
assertRecomputed(tfcCalc2.tfc, 0, "thread 2 should recompute");
assertTrue("Both results should be the same (as their inputs are the same objects)",
tfcCalc1.tfc == tfcCalc2.tfc);
r.close();
tr.close();
}
/**
* Simple test to make sure the TotalFacetCountsManager updates the
* TotalFacetCounts array only when it is supposed to, and whether it
* is recomputed or read from disk, but this time with TWO different
* TotalFacetCounts
*/
@Test
public void testMultipleIndices() throws IOException {
// Create temporary RAMDirectories
Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(2);
// Create our index/taxonomy writers
IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
DefaultFacetIndexingParams iParams = new DefaultFacetIndexingParams();
// Add a facet to the index
addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
addFacets(iParams, writers[1].indexWriter, writers[1].taxWriter, "d", "e");
// Commit Changes
writers[0].indexWriter.commit();
writers[0].taxWriter.commit();
writers[1].indexWriter.commit();
writers[1].taxWriter.commit();
// Open two readers
IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
// As this is the first time we have invoked the TotalFacetCountsManager, we
// should expect to compute.
TotalFacetCounts totalCounts0 =
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
int prevGen = -1;
prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 1");
assertTrue("attempt 1b for same input [0] shout find it in cache",
totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null));
// 2nd Reader - As this is the first time we have invoked the
// TotalFacetCountsManager, we should expect a state of NEW to be returned.
TotalFacetCounts totalCounts1 =
TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams, null);
prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 2");
assertTrue("attempt 2b for same input [1] shout find it in cache",
totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams, null));
// Right now cache size is one, so first TFC is gone and should be recomputed
totalCounts0 =
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 3");
// Similarly will recompute the second result
totalCounts1 =
TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams, null);
prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 4");
// Now we set the cache size to two, meaning both should exist in the
// cache simultaneously
TFC.setCacheSize(2);
// Re-compute totalCounts0 (was evicted from the cache when the cache was smaller)
totalCounts0 =
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null);
prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 5");
// now both are in the larger cache and should not be recomputed
totalCounts1 = TFC.getTotalCounts(readers[1].indexReader,
readers[1].taxReader, iParams, null);
assertTrue("with cache of size 2 res no. 0 should come from cache",
totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams, null));
assertTrue("with cache of size 2 res no. 1 should come from cache",
totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams, null));
writers[0].close();
writers[1].close();
readers[0].close();
readers[1].close();
for (Directory[] dirset : dirs) {
IOUtils.close(dirset);
}
}
}