Examples of org.apache.blur.trace.Tracer

org.apache.blur.trace.Tracer

        getScoreType(query.scoreType), context);
  }


  public static void populateSelector(IndexSearcherClosable searcher, String shardName, String table, Selector selector)
      throws IOException {
    Tracer trace = Trace.trace("populate selector");
    String rowId = selector.rowId;
    String recordId = selector.recordId;
    try {
      BooleanQuery query = new BooleanQuery();
      if (selector.recordOnly) {
        query.add(new TermQuery(new Term(RECORD_ID, recordId)), Occur.MUST);
        query.add(new TermQuery(new Term(ROW_ID, rowId)), Occur.MUST);
      } else {
        query.add(new TermQuery(new Term(ROW_ID, rowId)), Occur.MUST);
        query.add(new TermQuery(BlurUtil.PRIME_DOC_TERM), Occur.MUST);
      }
      TopDocs topDocs = searcher.search(query, 1);
      if (topDocs.totalHits > 1) {
        if (selector.recordOnly) {
          LOG.warn("Rowid [" + rowId + "], recordId [" + recordId
              + "] has more than one prime doc that is not deleted.");
        } else {
          LOG.warn("Rowid [" + rowId + "] has more than one prime doc that is not deleted.");
        }
      }
      if (topDocs.totalHits == 1) {
        selector.setLocationId(shardName + "/" + topDocs.scoreDocs[0].doc);
      } else {
        selector.setLocationId(NOT_FOUND);
      }
    } finally {
      trace.done();
    }
  }

View Full Code Here

        String shard = MutationHelper.getShardName(table, rowId, getNumberOfShards(table), _blurPartitioner);
        BlurIndex index = getBlurIndex(table, shard);
        map.put(shard, index);
        blurIndexes = map;
      }
      Tracer trace = Trace.trace("query setup", Trace.param("table", table));
      ShardServerContext shardServerContext = ShardServerContext.getShardServerContext();
      ParallelCall<Entry<String, BlurIndex>, BlurResultIterable> call;
      TableContext context = getTableContext(table);
      FieldManager fieldManager = context.getFieldManager();
      org.apache.blur.thrift.generated.Query simpleQuery = blurQuery.query;
      ReadInterceptor interceptor = context.getReadInterceptor();
      Filter readFilter = interceptor.getFilter();
      if (rowId != null) {
        if (simpleQuery.recordFilter == null) {
          simpleQuery.recordFilter = "+" + BlurConstants.ROW_ID + ":" + rowId;
        } else {
          simpleQuery.recordFilter = "+" + BlurConstants.ROW_ID + ":" + rowId + " +(" + simpleQuery.recordFilter + ")";
        }
      }
      Filter recordFilterForSearch = QueryParserUtil.parseFilter(table, simpleQuery.recordFilter, false, fieldManager,
          _filterCache, context);
      Filter rowFilterForSearch = QueryParserUtil.parseFilter(table, simpleQuery.rowFilter, true, fieldManager,
          _filterCache, context);
      Filter docFilter;
      if (recordFilterForSearch == null && readFilter != null) {
        docFilter = readFilter;
      } else if (recordFilterForSearch != null && readFilter == null) {
        docFilter = recordFilterForSearch;
      } else if (recordFilterForSearch != null && readFilter != null) {
        // @TODO dangerous call because of the bitsets that booleanfilter
        // creates.
        BooleanFilter booleanFilter = new BooleanFilter();
        booleanFilter.add(recordFilterForSearch, Occur.MUST);
        booleanFilter.add(readFilter, Occur.MUST);
        docFilter = booleanFilter;
      } else {
        docFilter = null;
      }
      Query userQuery = QueryParserUtil.parseQuery(simpleQuery.query, simpleQuery.rowQuery, fieldManager,
          rowFilterForSearch, docFilter, getScoreType(simpleQuery.scoreType), context);


      Query facetedQuery;
      FacetExecutor executor = null;
      if (blurQuery.facets != null) {
        long[] facetMinimums = getFacetMinimums(blurQuery.facets);
        executor = new FacetExecutor(blurQuery.facets.size(), facetMinimums, facetedCounts, running);
        facetedQuery = new FacetQuery(userQuery, getFacetQueries(blurQuery, fieldManager, context, rowFilterForSearch,
            recordFilterForSearch), executor);
      } else {
        facetedQuery = userQuery;
      }


      Sort sort = getSort(blurQuery, fieldManager);
      call = new SimpleQueryParallelCall(running, table, status, facetedQuery, blurQuery.selector,
          _queriesInternalMeter, shardServerContext, runSlow, _fetchCount, _maxHeapPerRowFetch,
          context.getSimilarity(), context, sort, _deepPagingCache);
      trace.done();
      MergerBlurResultIterable merger = new MergerBlurResultIterable(blurQuery);
      BlurResultIterable merge = ForkJoin.execute(_executor, blurIndexes.entrySet(), call, new Cancel() {
        @Override
        public void cancel() {
          running.set(false);

View Full Code Here

          && selector.columnFamiliesToFetch.isEmpty() && selector.columnsToFetch.isEmpty()) {
        // exit early
        returnIdsOnly = true;
      }


      Tracer t1 = Trace.trace("fetchRow - live docs");
      Bits liveDocs = MultiFields.getLiveDocs(reader);
      t1.done();
      ResetableDocumentStoredFieldVisitor fieldVisitor = getFieldSelector(selector);
      if (selector.isRecordOnly()) {
        // select only the row for the given data or location id.
        if (isFiltered(docId, reader, filter)) {
          fetchResult.exists = false;
          fetchResult.deleted = false;
          return;
        } else if (liveDocs != null && !liveDocs.get(docId)) {
          fetchResult.exists = false;
          fetchResult.deleted = true;
          return;
        } else {
          fetchResult.exists = true;
          fetchResult.deleted = false;
          reader.document(docId, fieldVisitor);
          Document document = fieldVisitor.getDocument();
          if (highlightQuery != null && fieldManager != null) {
            HighlightOptions highlightOptions = selector.getHighlightOptions();
            String preTag = highlightOptions.getPreTag();
            String postTag = highlightOptions.getPostTag();
            try {
              document = HighlightHelper.highlight(docId, document, highlightQuery, fieldManager, reader, preTag,
                  postTag);
            } catch (InvalidTokenOffsetsException e) {
              LOG.error("Unknown error while tring to highlight", e);
            }
          }
          fieldVisitor.reset();
          fetchResult.recordResult = getRecord(document);
          return;
        }
      } else {
        Tracer trace = Trace.trace("fetchRow - Row read");
        try {
          if (liveDocs != null && !liveDocs.get(docId)) {
            fetchResult.exists = false;
            fetchResult.deleted = true;
            return;
          } else {
            fetchResult.exists = true;
            fetchResult.deleted = false;
            if (returnIdsOnly) {
              String rowId = selector.getRowId();
              if (rowId == null) {
                rowId = getRowId(reader, docId);
              }
              fetchResult.rowResult = new FetchRowResult();
              fetchResult.rowResult.row = new Row(rowId, null);
            } else {
              List<Document> docs;
              AtomicBoolean moreDocsToFetch = new AtomicBoolean(false);
              AtomicInteger totalRecords = new AtomicInteger();
              BlurHighlighter highlighter = new BlurHighlighter(highlightQuery, fieldManager, selector);
              Tracer docTrace = Trace.trace("fetchRow - Document read");
              docs = BlurUtil.fetchDocuments(reader, fieldVisitor, selector, maxHeap, table + "/" + shard,
                  tableContext.getDefaultPrimeDocTerm(), filter, moreDocsToFetch, totalRecords, highlighter);
              docTrace.done();
              Tracer rowTrace = Trace.trace("fetchRow - Row create");
              Row row = getRow(docs);
              if (row == null) {
                String rowId = selector.getRowId();
                if (rowId == null) {
                  rowId = getRowId(reader, docId);
                }
                row = new Row(rowId, null);
              }
              fetchResult.rowResult = new FetchRowResult(row, selector.getStartRecord(),
                  selector.getMaxRecordsToFetch(), moreDocsToFetch.get(), totalRecords.get());
              rowTrace.done();
            }
            return;
          }
        } finally {
          trace.done();

View Full Code Here


  @Override
  public BlurResults query(final String table, final BlurQuery blurQuery) throws BlurException, TException {
    try {
      checkTable(table);
      Tracer trace = Trace.trace("query - setup", Trace.param("table", table), Trace.param("blurQuery", blurQuery));
      String cluster = _clusterStatus.getCluster(true, table);
      _queryChecker.checkQuery(blurQuery);
      checkSelectorFetchSize(blurQuery.getSelector());
      TableDescriptor tableDescriptor = _clusterStatus.getTableDescriptor(true, cluster, table);
      int shardCount = tableDescriptor.getShardCount();
      if (blurQuery.getUuid() == null) {
        blurQuery.setUuid(UUID.randomUUID().toString());
      }
      BlurUtil.setStartTime(blurQuery);
      trace.done();


      BlurUtil.setStartTime(blurQuery);


      OUTER: for (int retries = 0; retries < _maxDefaultRetries; retries++) {
        Tracer selectorTrace = Trace.trace("selector - setup", Trace.param("retries", retries));
        final AtomicLongArray facetCounts = BlurUtil.getAtomicLongArraySameLengthAsList(blurQuery.facets);
        Selector selector = blurQuery.getSelector();
        if (selector == null) {
          selector = new Selector();
          selector.setColumnFamiliesToFetch(EMPTY_SET);
          selector.setColumnsToFetch(EMPTY_MAP);
          if (!blurQuery.query.rowQuery) {
            selector.setRecordOnly(true);
          }
        } else {
          HighlightOptions highlightOptions = selector.getHighlightOptions();
          if (highlightOptions != null && highlightOptions.getQuery() == null) {
            highlightOptions.setQuery(blurQuery.getQuery());
          }
        }
        blurQuery.setSelector(null);
        selectorTrace.done();


        BlurCommand<BlurResultIterable> command = new BlurCommand<BlurResultIterable>() {
          @Override
          public BlurResultIterable call(Client client, Connection connection) throws BlurException, TException {
            return new BlurResultIterableClient(connection, client, table, blurQuery, facetCounts, _remoteFetchCount);
          }


          @Override
          public BlurResultIterable call(Client client) throws BlurException, TException {
            throw new RuntimeException("Won't be called.");
          }
        };


        command.setDetachClient(true);


        MergerBlurResultIterable merger = new MergerBlurResultIterable(blurQuery);
        BlurResultIterable hitsIterable = null;
        try {
          String rowId = blurQuery.getRowId();
          if (rowId == null) {
            Tracer scatterGatherTrace = Trace.trace("query - scatterGather", Trace.param("retries", retries));
            try {
              hitsIterable = scatterGather(tableDescriptor.getCluster(), command, merger);
            } finally {
              scatterGatherTrace.done();
            }
          } else {
            String clientHostnamePort = getNode(table, rowId);
            hitsIterable = _client.execute(clientHostnamePort, command, _maxFetchRetries, _fetchDelay, _maxFetchDelay);
          }
          BlurResults results;
          Tracer convertToBlurResults = Trace.trace("query - convertToBlurResults", Trace.param("retries", retries));
          try {
            results = convertToBlurResults(hitsIterable, blurQuery, facetCounts, _executor, selector, table);
          } finally {
            convertToBlurResults.done();
          }
          if (!validResults(results, shardCount, blurQuery)) {
            BlurClientManager.sleep(_defaultDelay, _maxDefaultDelay, retries, _maxDefaultRetries);
            Map<String, String> map = getTableLayout(table);
            LOG.info("Current layout for table [{0}] is [{1}]", table, map);

View Full Code Here

    if (facetCounts != null) {
      results.facetCounts = BlurUtil.toList(facetCounts);
    }
    if (selector != null) {


      Tracer trace = Trace.trace("fetch data", Trace.param("table", table));


      // Gather client objects and build batches for fetching.
      IdentityHashMap<Client, List<Selector>> map = new IdentityHashMap<Client, List<Selector>>();


      // Need to maintain original order.
      final IdentityHashMap<Selector, Integer> indexMap = new IdentityHashMap<Selector, Integer>();
      for (int i = 0; i < results.results.size(); i++) {
        final LazyBlurResult result = (LazyBlurResult) results.results.get(i);
        Client client = result.getClient();
        Selector s = new Selector(selector);
        s.setLocationId(result.locationId);
        List<Selector> list = map.get(client);
        if (list == null) {
          list = new ArrayList<Selector>();
          map.put(client, list);
        }
        list.add(s);
        indexMap.put(s, i);
      }


      // Execute batch fetches
      List<Future<Boolean>> futures = new ArrayList<Future<Boolean>>();
      final AtomicReferenceArray<FetchResult> fetchResults = new AtomicReferenceArray<FetchResult>(
          results.results.size());
      for (Entry<Client, List<Selector>> entry : map.entrySet()) {
        final Client client = entry.getKey();
        final List<Selector> list = entry.getValue();
        futures.add(executor.submit(new Callable<Boolean>() {
          @Override
          public Boolean call() throws Exception {
            Tracer trace = Trace.trace("remote call - thrift", Trace.param("node", getNode(client)));
            try {
              TraceId traceId = Trace.getTraceId();
              if (traceId != null) {
                client.startTrace(traceId.getRootId(), traceId.getRequestId());
              }
              List<FetchResult> fetchRowBatch = client.fetchRowBatch(table, list);
              for (int i = 0; i < list.size(); i++) {
                int index = indexMap.get(list.get(i));
                fetchResults.set(index, fetchRowBatch.get(i));
              }
              return Boolean.TRUE;
            } finally {
              trace.done();
            }
          }


          private String getNode(Client client) {
            TProtocol inputProtocol = client.getInputProtocol();
            TTransport transport = inputProtocol.getTransport();
            if (transport instanceof TFramedTransport) {
              TFramedTransport framedTransport = (TFramedTransport) transport;
              transport = framedTransport.getTransport();
            }
            if (transport instanceof TSocket) {
              TSocket tsocket = (TSocket) transport;
              Socket socket = tsocket.getSocket();
              SocketAddress remoteSocketAddress = socket.getRemoteSocketAddress();
              return remoteSocketAddress.toString();
            }
            return "unknown";
          }
        }));
      }


      // Wait for all parallel calls to finish.
      for (Future<Boolean> future : futures) {
        try {
          future.get();
        } catch (ExecutionException e) {
          Throwable throwable = e.getCause();
          if (throwable instanceof BlurException) {
            throw (BlurException) throwable;
          } else {
            throw new BException("Unknown error during fetch", throwable);
          }
        }
      }


      // Place fetch results into result object for response.
      for (int i = 0; i < fetchResults.length(); i++) {
        FetchResult fetchResult = fetchResults.get(i);
        BlurResult result = results.results.get(i);
        result.setFetchResult(fetchResult);
        result.setLocationId(null);
      }


      trace.done();
    }
    results.query = query;
    results.query.selector = selector;
    return results;
  }

View Full Code Here

    public <T> T execute(final String node, final BlurCommand<T> command, final int maxRetries, final long backOffTime,
        final long maxBackOffTime) throws BlurException, TException, IOException {
      Callable<T> callable = Trace.getCallable(new Callable<T>() {
        @Override
        public T call() throws Exception {
          Tracer trace = Trace.trace("remote call - thrift", Trace.param("node", node));
          try {
            return BlurClientManager.execute(node + "#" + _timeout, command, maxRetries, backOffTime, maxBackOffTime);
          } finally {
            trace.done();
          }
        }
      });
      try {
        return callable.call();

View Full Code Here

    return new Lock() {
      private boolean _set;


      @Override
      public boolean obtain() throws IOException {
        Tracer trace = Trace.trace("filesystem - obtain", Trace.param("lockPath", lockPath));
        try {
          if (_set) {
            throw new IOException("Lock for [" + _baseLockKey + "] can only be set once.");
          }
          try {
            _lockKey = (_baseLockKey + "/" + System.currentTimeMillis()).getBytes();
            FSDataOutputStream outputStream = _fileSystem.create(lockPath, true);
            outputStream.write(_lockKey);
            outputStream.close();
          } finally {
            _set = true;
          }
          return true;
        } finally {
          trace.done();
        }
      }


      @Override
      public void release() throws IOException {
        Tracer trace = Trace.trace("filesystem - release",Trace.param("lockPath", lockPath));
        try {
          _fileSystem.delete(lockPath, false);
        } finally {
          trace.done();
        }
      }


      @Override
      public boolean isLocked() throws IOException {
        if (_lastCheck.get() + _checkTime >= System.currentTimeMillis()) {
          return true;
        }
        Tracer trace = Trace.trace("filesystem - isLocked", Trace.param("lockPath", lockPath));
        try {
          if (!_set) {
            LOG.info("The lock has NOT been set.");
            return false;
          }
          if (!_fileSystem.exists(lockPath)) {
            LOG.info("The lock file has been removed.");
            return false;
          }
          FileStatus fileStatus = _fileSystem.getFileStatus(lockPath);
          long len = fileStatus.getLen();
          if (len != _lockKey.length) {
            LOG.info("The lock file length has changed.");
            return false;
          }
          byte[] buf = new byte[_lockKey.length];
          FSDataInputStream inputStream = _fileSystem.open(lockPath);
          inputStream.readFully(buf);
          inputStream.close();
          if (Arrays.equals(_lockKey, buf)) {
            _lastCheck.set(System.currentTimeMillis());
            return true;
          }
          LOG.info("The lock information has been changed.");
          return false;
        } finally {
          trace.done();
        }
      }
    };
  }

View Full Code Here

    return new Lock() {
      private boolean _set;


      @Override
      public boolean obtain() throws IOException {
        Tracer trace = Trace.trace("filesystem - release", Trace.param("lockPath", lockPath));
        try {
          if (_set) {
            throw new IOException("Lock for [" + _baseLockKey + "] can only be set once.");
          }
          try {
            _lockKey = (_baseLockKey + "/" + System.currentTimeMillis()).getBytes();
            FSDataOutputStream outputStream = _fileSystem.create(lockPath, true);
            outputStream.write(_lockKey);
            outputStream.close();
          } finally {
            _set = true;
          }
          return true;
        } finally {
          trace.done();
        }
      }


      @Override
      public void release() throws IOException {
        Tracer trace = Trace.trace("filesystem - release", Trace.param("lockPath", lockPath));
        try {
          _fileSystem.delete(lockPath, false);
        } finally {
          trace.done();
        }
      }


      @Override
      public boolean isLocked() throws IOException {
        Tracer trace = Trace.trace("filesystem - isLocked", Trace.param("lockPath", lockPath));
        try {
          if (!_set) {
            return false;
          }
          if (!_fileSystem.exists(lockPath)) {
            return false;
          }
          FileStatus fileStatus = _fileSystem.getFileStatus(lockPath);
          long len = fileStatus.getLen();
          if (len != _lockKey.length) {
            return false;
          }
          byte[] buf = new byte[_lockKey.length];
          FSDataInputStream inputStream = _fileSystem.open(lockPath);
          inputStream.readFully(buf);
          inputStream.close();
          if (Arrays.equals(_lockKey, buf)) {
            return true;
          }
          return false;
        } finally {
          trace.done();
        }
      }
    };
  }

View Full Code Here

        String argsStr = null;
        long start = System.nanoTime();
        String name = method.getName();
        boolean error = false;
        LoggerArgsState loggerArgsState = null;
        Tracer trace = Trace.trace("thrift recv", Trace.param("method", method.getName()),
            Trace.param("connection", tracingConnectionString));
        User user = UserContext.getUser();
        boolean notSetUserMethod = isNotSetUserMethod(name);
        try {
          if (REQUEST_LOG.isInfoEnabled() && notSetUserMethod) {
            if (argsStr == null) {
              loggerArgsState = _loggerArgsState.get();
              argsStr = getArgsStr(args, name, loggerArgsState);
            }
            REQUEST_LOG.info(getRequestLogMessage(requestId, connectionString, argsStr, name, user));
          }
          return method.invoke(t, args);
        } catch (InvocationTargetException e) {
          error = true;
          throw e.getTargetException();
        } finally {
          trace.done();
          long end = System.nanoTime();
          double ms = (end - start) / 1000000.0;
          if (RESPONSE_LOG.isInfoEnabled() && notSetUserMethod) {
            if (argsStr == null) {
              if (loggerArgsState == null) {

View Full Code Here

      throws InterruptedException, ExecutionException, BlurException {
    BlurResults results = new BlurResults();
    results.setTotalResults(hitsIterable.getTotalResults());
    results.setShardInfo(hitsIterable.getShardInfo());
    if (query.minimumNumberOfResults > 0) {
      Tracer trace = Trace.trace("convertToHits - skipTo", Trace.param("start", query.start));
      hitsIterable.skipTo(query.start);
      trace.done();
      BlurIterator<BlurResult, BlurException> iterator = hitsIterable.iterator();
      for (int count = 0; count < query.fetch && iterator.hasNext(); count++) {
        results.addToResults(iterator.next());
      }
    }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.blur.trace.Tracer

org.apache.blur.analysis.HdfsFieldManager

org.apache.blur.index.OptimisticLockFactory

org.apache.blur.lucene.search.FacetExecutor

org.apache.blur.lucene.search.FacetExecutor$Info

org.apache.blur.manager.IndexManager

org.apache.blur.manager.IndexManager$SimpleQueryParallelCall

org.apache.blur.manager.indexserver.AbstractDistributedIndexServer

org.apache.blur.manager.results.BlurResultIterableMultiple

org.apache.blur.manager.writer.BlurIndexSimpleWriter

org.apache.blur.server.IndexSearcherClosable

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.