Package org.elasticsearch.index.fielddata.ordinals

Examples of org.elasticsearch.index.fielddata.ordinals.Ordinals


                values = BigArrays.NON_RECYCLING_INSTANCE.grow(values, numTerms + 1);
                values.set(numTerms++, NumericUtils.sortableIntToFloat(NumericUtils.prefixCodedToInt(term)));
            }
            values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms);
            final FloatArray finalValues = values;
            final Ordinals build = builder.build(fieldDataType.getSettings());
            RandomAccessOrds ordinals = build.ordinals();
            if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
                final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
                data = new AtomicDoubleFieldData(ramBytesUsed) {

                    @Override
                    public SortedNumericDoubleValues getDoubleValues() {
                        return withOrdinals(build, finalValues, reader.maxDoc());
                    }
                   
                    @Override
                    public Iterable<? extends Accountable> getChildResources() {
                        List<Accountable> resources = new ArrayList<>();
                        resources.add(Accountables.namedAccountable("ordinals", build));
                        resources.add(Accountables.namedAccountable("values", finalValues));
                        return Collections.unmodifiableList(resources);
                    }

                };
            } else {
                final BitSet set = builder.buildDocsWithValuesSet();

                // there's sweet spot where due to low unique value count, using ordinals will consume less memory
                long singleValuesArraySize = reader.maxDoc() * RamUsageEstimator.NUM_BYTES_FLOAT + (set == null ? 0 : set.ramBytesUsed());
                long uniqueValuesArraySize = values.ramBytesUsed();
                long ordinalsSize = build.ramBytesUsed();
                if (uniqueValuesArraySize + ordinalsSize < singleValuesArraySize) {
                    final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
                    success = true;
                    return data = new AtomicDoubleFieldData(ramBytesUsed) {

                        @Override
                        public SortedNumericDoubleValues getDoubleValues() {
View Full Code Here


                }
                lat.set(ord, encoding.encodeCoordinate(point.getLat()));
                lon.set(ord, encoding.encodeCoordinate(point.getLon()));
            }

            Ordinals build = builder.build(fieldDataType.getSettings());
            RandomAccessOrds ordinals = build.ordinals();
            if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
                if (lat.size() != ordinals.getValueCount()) {
                    lat = lat.resize(ordinals.getValueCount());
                    lon = lon.resize(ordinals.getValueCount());
                }
View Full Code Here

                ++numTerms;
            }
            lat = BigArrays.NON_RECYCLING_INSTANCE.resize(lat, numTerms);
            lon = BigArrays.NON_RECYCLING_INSTANCE.resize(lon, numTerms);

            Ordinals build = builder.build(fieldDataType.getSettings());
            RandomAccessOrds ordinals = build.ordinals();
            if (!(FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) {
                int maxDoc = reader.maxDoc();
                DoubleArray sLat = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
                DoubleArray sLon = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(reader.maxDoc());
                for (int i = 0; i < maxDoc; i++) {
View Full Code Here

                values = BigArrays.NON_RECYCLING_INSTANCE.grow(values, numTerms + 1);
                values.set(numTerms++, NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(term)));
            }
            values = BigArrays.NON_RECYCLING_INSTANCE.resize(values, numTerms);
            final DoubleArray finalValues = values;
            final Ordinals build = builder.build(fieldDataType.getSettings());
            RandomAccessOrds ordinals = build.ordinals();
            if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
                final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
                data = new AtomicDoubleFieldData(ramBytesUsed) {

                    @Override
                    public SortedNumericDoubleValues getDoubleValues() {
                        return withOrdinals(build, finalValues, reader.maxDoc());
                    }
                   
                    @Override
                    public Iterable<? extends Accountable> getChildResources() {
                        List<Accountable> resources = new ArrayList<>();
                        resources.add(Accountables.namedAccountable("ordinals", build));
                        resources.add(Accountables.namedAccountable("values", finalValues));
                        return Collections.unmodifiableList(resources);
                    }

                };
            } else {
                final BitSet set = builder.buildDocsWithValuesSet();

                // there's sweet spot where due to low unique value count, using ordinals will consume less memory
                long singleValuesArraySize = reader.maxDoc() * RamUsageEstimator.NUM_BYTES_DOUBLE + (set == null ? 0 : set.ramBytesUsed());
                long uniqueValuesArraySize = values.ramBytesUsed();
                long ordinalsSize = build.ramBytesUsed();
                if (uniqueValuesArraySize + ordinalsSize < singleValuesArraySize) {
                    final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
                    success = true;
                    return data = new AtomicDoubleFieldData(ramBytesUsed) {

                        @Override
                        public SortedNumericDoubleValues getDoubleValues() {
View Full Code Here

                }
            }

            FST<Long> fst = fstBuilder.finish();

            final Ordinals ordinals = builder.build(fieldDataType.getSettings());

            data = new FSTBytesAtomicFieldData(fst, ordinals);
            success = true;
            return data;
        } finally {
View Full Code Here

                }

                ImmutableOpenMap.Builder<String, AtomicOrdinalsFieldData> typeToAtomicFieldData = ImmutableOpenMap.builder(typeBuilders.size());
                for (ObjectObjectCursor<String, TypeBuilder> cursor : typeBuilders) {
                    PagedBytes.Reader bytesReader = cursor.value.bytes.freeze(true);
                    final Ordinals ordinals = cursor.value.builder.build(fieldDataType.getSettings());

                    typeToAtomicFieldData.put(
                            cursor.key,
                            new PagedBytesAtomicFieldData(bytesReader, cursor.value.termOrdToBytesOffset.build(), ordinals)
                    );
View Full Code Here

                        ? NumericUtils.prefixCodedToLong(term)
                        : NumericUtils.prefixCodedToInt(term);
                valuesBuilder.add(value);
            }
            final PackedLongValues values = valuesBuilder.build();
            final Ordinals build = builder.build(fieldDataType.getSettings());
            CommonSettings.MemoryStorageFormat formatHint = CommonSettings.getMemoryStorageHint(fieldDataType);

            RandomAccessOrds ordinals = build.ordinals();
            if (FieldData.isMultiValued(ordinals) || formatHint == CommonSettings.MemoryStorageFormat.ORDINALS) {
                final long ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
                data = new AtomicLongFieldData(ramBytesUsed) {

                    @Override
                    public SortedNumericDocValues getLongValues() {
                        return withOrdinals(build, values, reader.maxDoc());
                    }

                    @Override
                    public Iterable<? extends Accountable> getChildResources() {
                        List<Accountable> resources = new ArrayList<>();
                        resources.add(Accountables.namedAccountable("ordinals", build));
                        resources.add(Accountables.namedAccountable("values", values));
                        return Collections.unmodifiableList(resources);
                    }
                };
            } else {
                final BitSet docsWithValues = builder.buildDocsWithValuesSet();

                long minV, maxV;
                minV = maxV = 0;
                if (values.size() > 0) {
                    minV = values.get(0);
                    maxV = values.get(values.size() - 1);
                }


                final float acceptableOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_overhead_ratio", PackedInts.DEFAULT);
                final int pageSize = fieldDataType.getSettings().getAsInt("single_value_page_size", 1024);

                if (formatHint == null) {
                    formatHint = chooseStorageFormat(reader, values, build, ordinals, minV, maxV, acceptableOverheadRatio, pageSize);
                }

                logger.trace("single value format for field [{}] set to [{}]", getFieldNames().fullName(), formatHint);

                switch (formatHint) {
                    case PACKED:
                        // Encode document without a value with a special value
                        long missingV = 0;
                        if (docsWithValues != null) {
                            if ((maxV - minV + 1) == values.size()) {
                                // values are dense
                                if (minV > Long.MIN_VALUE) {
                                    missingV = --minV;
                                } else {
                                    assert maxV != Long.MAX_VALUE;
                                    missingV = ++maxV;
                                }
                            } else {
                                for (long i = 1; i < values.size(); ++i) {
                                    if (values.get(i) > values.get(i - 1) + 1) {
                                        missingV = values.get(i - 1) + 1;
                                        break;
                                    }
                                }
                            }
                            missingV -= minV;
                        }
                        final long missingValue = missingV;
                        final long minValue = minV;
                        final long maxValue = maxV;

                        final long valuesDelta = maxValue - minValue;
                        int bitsRequired = valuesDelta < 0 ? 64 : PackedInts.bitsRequired(valuesDelta);
                        final PackedInts.Mutable sValues = PackedInts.getMutable(reader.maxDoc(), bitsRequired, acceptableOverheadRatio);

                        if (docsWithValues != null) {
                            sValues.fill(0, sValues.size(), missingV);
                        }

                        for (int i = 0; i < reader.maxDoc(); i++) {
                            ordinals.setDocument(i);
                            if (ordinals.cardinality() > 0) {
                                final long ord = ordinals.ordAt(0);
                                long value = values.get(ord);
                                sValues.set(i, value - minValue);
                            }
                        }
                        long ramBytesUsed = values.ramBytesUsed() + (docsWithValues == null ? 0 : docsWithValues.ramBytesUsed());
                        data = new AtomicLongFieldData(ramBytesUsed) {

                            @Override
                            public SortedNumericDocValues getLongValues() {
                                if (docsWithValues == null) {
                                    return singles(sValues, minValue);
                                } else {
                                    return sparseSingles(sValues, minValue, missingValue, reader.maxDoc());
                                }
                            }
                           
                            @Override
                            public Iterable<? extends Accountable> getChildResources() {
                                List<Accountable> resources = new ArrayList<>();
                                resources.add(Accountables.namedAccountable("values", sValues));
                                if (docsWithValues != null) {
                                    resources.add(Accountables.namedAccountable("missing bitset", docsWithValues));
                                }
                                return Collections.unmodifiableList(resources);
                            }

                        };
                        break;
                    case PAGED:
                        final PackedLongValues.Builder dpValues = PackedLongValues.deltaPackedBuilder(pageSize, acceptableOverheadRatio);

                        long lastValue = 0;
                        for (int i = 0; i < reader.maxDoc(); i++) {
                            ordinals.setDocument(i);
                            if (ordinals.cardinality() > 0) {
                                final long ord = ordinals.ordAt(i);
                                lastValue = values.get(ord);
                            }
                            dpValues.add(lastValue);
                        }
                        final PackedLongValues pagedValues = dpValues.build();
                        ramBytesUsed = pagedValues.ramBytesUsed();
                        if (docsWithValues != null) {
                            ramBytesUsed += docsWithValues.ramBytesUsed();
                        }
                        data = new AtomicLongFieldData(ramBytesUsed) {

                            @Override
                            public SortedNumericDocValues getLongValues() {
                                return pagedSingles(pagedValues, docsWithValues);
                            }

                            @Override
                            public Iterable<? extends Accountable> getChildResources() {
                                List<Accountable> resources = new ArrayList<>();
                                resources.add(Accountables.namedAccountable("values", pagedValues));
                                if (docsWithValues != null) {
                                    resources.add(Accountables.namedAccountable("missing bitset", docsWithValues));
                                }
                                return Collections.unmodifiableList(resources);
                            }
                           
                        };
                        break;
                    case ORDINALS:
                        ramBytesUsed = build.ramBytesUsed() + values.ramBytesUsed();
                        data = new AtomicLongFieldData(ramBytesUsed) {

                            @Override
                            public SortedNumericDocValues getLongValues() {
                                return withOrdinals(build, values, reader.maxDoc());
View Full Code Here

                for (int docId = docsEnum.nextDoc(); docId != DocsEnum.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
                    builder.addDoc(docId);
                }
            }
            PagedBytes.Reader bytesReader = bytes.freeze(true);
            final Ordinals ordinals = builder.build(fieldDataType.getSettings());

            data = new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffset.build(), ordinals);
            success = true;
            return data;
        } finally {
View Full Code Here

TOP

Related Classes of org.elasticsearch.index.fielddata.ordinals.Ordinals

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.