Examples of MatrixSlice

org.apache.mahout.math.MatrixSlice

Examples of org.apache.mahout.math.MatrixSlice


    StringBuilder prefsAsText = new StringBuilder();
    String separator = "";
    Iterator<MatrixSlice> sliceIterator = preferences.iterateAll();
    while (sliceIterator.hasNext()) {
      MatrixSlice slice = sliceIterator.next();
      Iterator<Vector.Element> elementIterator = slice.vector().iterateNonZero();
      while (elementIterator.hasNext()) {
        Vector.Element e = elementIterator.next();
        if (!Double.isNaN(e.get())) {
          prefsAsText.append(separator).append(slice.index()).append(',').append(e.index()).append(',').append(e.get());
          separator = "\n";
        }
      }
    }
    logger.info("Input matrix:\n" + prefsAsText);
    writeLines(inputFile, prefsAsText.toString());


    ParallelALSFactorizationJob alsFactorization = new ParallelALSFactorizationJob();


    Configuration conf = new Configuration();
    conf.set("mapred.input.dir", inputFile.getAbsolutePath());
    conf.set("mapred.output.dir", outputDir.getAbsolutePath());
    conf.setBoolean("mapred.output.compress", false);


    alsFactorization.setConf(conf);
    int numFeatures = 3;
    int numIterations = 5;
    double lambda = 0.065;
    alsFactorization.run(new String[] { "--tempDir", tmpDir.getAbsolutePath(), "--lambda", String.valueOf(lambda),
        "--numFeatures", String.valueOf(numFeatures), "--numIterations", String.valueOf(numIterations) });


    Matrix u = MathHelper.readEntries(conf, new Path(outputDir.getAbsolutePath(), "U/part-r-00000"),
        preferences.numRows(), numFeatures);
    Matrix m = MathHelper.readEntries(conf, new Path(outputDir.getAbsolutePath(), "M/part-r-00000"),
      preferences.numCols(), numFeatures);


    RunningAverage avg = new FullRunningAverage();
    sliceIterator = preferences.iterateAll();
    while (sliceIterator.hasNext()) {
      MatrixSlice slice = sliceIterator.next();
      Iterator<Vector.Element> elementIterator = slice.vector().iterateNonZero();
      while (elementIterator.hasNext()) {
        Vector.Element e = elementIterator.next();
        if (!Double.isNaN(e.get())) {
          double pref = e.get();
          double estimate = u.getRow(slice.index()).dot(m.getRow(e.index()));
          double err = pref - estimate;
          avg.addDatum(err * err);
          logger.info("Comparing preference of user [" + slice.index() + "] towards item [" + e.index() + "], " +
              "was [" + pref + "] estimate is [" + estimate + ']');
        }
      }
    }
    double rmse = Math.sqrt(avg.getAverage());

View Full Code Here

Examples of org.apache.mahout.math.MatrixSlice

    Iterator<MatrixSlice> mIt = m.iterateAll();
    Iterator<MatrixSlice> mttIt = mtt.iterateAll();
    Map<Integer, Vector> mMap = new HashMap<Integer,Vector>();
    Map<Integer, Vector> mttMap = new HashMap<Integer, Vector>();
    while (mIt.hasNext() && mttIt.hasNext()) {
      MatrixSlice ms = mIt.next();
      mMap.put(ms.index(), ms.vector());
      MatrixSlice mtts = mttIt.next();
      mttMap.put(mtts.index(), mtts.vector());
    }
    for(Map.Entry<Integer, Vector> entry : mMap.entrySet()) {
      Integer key = entry.getKey();
      Vector value = entry.getValue();
      if(value == null || mttMap.get(key) == null) {

View Full Code Here

Examples of org.apache.mahout.math.MatrixSlice

          public boolean hasNext() {
            return it.hasNext();
          }
          @Override
          public VectorWritable next() {
            MatrixSlice slice = it.next();
            return new VectorWritable(slice.vector());
          }
          @Override
          public void remove() {
            it.remove();
          }

View Full Code Here

Examples of org.apache.mahout.math.MatrixSlice

          new SequenceFileDirIterator<IntWritable,VectorWritable>(new Path(rowPath, "*"),
                                                                  PathType.GLOB, null, null, true, conf),
          new Function<Pair<IntWritable,VectorWritable>,MatrixSlice>() {
            @Override
            public MatrixSlice apply(Pair<IntWritable, VectorWritable> from) {
              return new MatrixSlice(from.getSecond().get(), from.getFirst().get());
            }
          });
    } catch (IOException ioe) {
      throw new IllegalStateException(ioe);
    }

View Full Code Here

Examples of org.apache.mahout.math.MatrixSlice

    FileSystem fs = FileSystem.get(conf);
    SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
    IntWritable iw = new IntWritable();
    int numEigensWritten = 0;
    for (Map.Entry<MatrixSlice, EigenStatus> pruneSlice : prunedEigenMeta) {
      MatrixSlice s = pruneSlice.getKey();
      EigenStatus meta = pruneSlice.getValue();
      EigenVector ev = new EigenVector(s.vector(),
                                       meta.getEigenValue(),
                                       Math.abs(1 - meta.getCosAngle()),
                                       s.index());
      log.info("appending {} to {}", ev, path);
      Writable vw = new VectorWritable(ev);
      iw.set(s.index());
      seqWriter.append(iw, vw);


      // increment the number of eigenvectors written and see if we've
      // reached our specified limit, or if we wish to write all eigenvectors
      // (latter is built-in, since numEigensWritten will always be > 0

View Full Code Here

Examples of org.apache.mahout.math.MatrixSlice

    try {
      IntWritable iw = new IntWritable();
      int numEigensWritten = 0;
      int index = 0;
      for (Map.Entry<MatrixSlice,EigenStatus> pruneSlice : prunedEigenMeta) {
        MatrixSlice s = pruneSlice.getKey();
        EigenStatus meta = pruneSlice.getValue();
        EigenVector ev = new EigenVector(s.vector(), meta.getEigenValue(), Math.abs(1 - meta.getCosAngle()), s.index());
        // log.info("appending {} to {}", ev, path);
        Writable vw = new VectorWritable(ev);
        iw.set(index++);
        seqWriter.append(iw, vw);

View Full Code Here

Examples of org.apache.mahout.math.MatrixSlice

    SVDRecommender svdRecommender = new SVDRecommender(dataModel, factorizer);


    RunningAverage avg = new FullRunningAverage();
    Iterator<MatrixSlice> sliceIterator = preferences.iterateAll();
    while (sliceIterator.hasNext()) {
      MatrixSlice slice = sliceIterator.next();
      for (Vector.Element e : slice.vector().all()) {


        long userID = slice.index() + 1;
        long itemID = e.index() + 1;


        if (!Double.isNaN(e.get())) {
          double pref = e.get();
          double estimate = svdRecommender.estimatePreference(userID, itemID);


          double confidence = 1 + alpha * observations.getQuick(slice.index(), e.index());
          double err = confidence * (pref - estimate) * (pref - estimate);
          avg.addDatum(err);
          log.info("Comparing preference of user [{}] towards item [{}], was [{}] with confidence [{}] "
              + "estimate is [{}]", slice.index(), e.index(), pref, confidence, estimate);
        }
      }
    }
    double rmse = Math.sqrt(avg.getAverage());
    log.info("RMSE: {}", rmse);

View Full Code Here

Examples of org.apache.mahout.math.MatrixSlice

    Iterator<MatrixSlice> mIt = m.iterateAll();
    Iterator<MatrixSlice> mttIt = mtt.iterateAll();
    Map<Integer, Vector> mMap = Maps.newHashMap();
    Map<Integer, Vector> mttMap = Maps.newHashMap();
    while (mIt.hasNext() && mttIt.hasNext()) {
      MatrixSlice ms = mIt.next();
      mMap.put(ms.index(), ms.vector());
      MatrixSlice mtts = mttIt.next();
      mttMap.put(mtts.index(), mtts.vector());
    }
    for (Map.Entry<Integer, Vector> entry : mMap.entrySet()) {
      Integer key = entry.getKey();
      Vector value = entry.getValue();
      if (value == null || mttMap.get(key) == null) {

View Full Code Here

Examples of org.apache.mahout.math.MatrixSlice

    SequenceFile.Writer seqWriter = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
    try {
      IntWritable iw = new IntWritable();
      int numEigensWritten = 0;
      for (Map.Entry<MatrixSlice, EigenStatus> pruneSlice : prunedEigenMeta) {
        MatrixSlice s = pruneSlice.getKey();
        EigenStatus meta = pruneSlice.getValue();
        EigenVector ev = new EigenVector(s.vector(),
                                         meta.getEigenValue(),
                                         Math.abs(1 - meta.getCosAngle()),
                                         s.index());
        //log.info("appending {} to {}", ev, path);
        Writable vw = new VectorWritable(ev);
        iw.set(s.index());
        seqWriter.append(iw, vw);


        // increment the number of eigenvectors written and see if we've
        // reached our specified limit, or if we wish to write all eigenvectors
        // (latter is built-in, since numEigensWritten will always be > 0

View Full Code Here

Examples of org.apache.mahout.math.MatrixSlice

    Iterator<MatrixSlice> docIterator = matrix.iterator();
    Iterator<MatrixSlice> docTopicIterator = docTopicCounts.iterator();
    double perplexity = 0;
    double matrixNorm = 0;
    while (docIterator.hasNext() && docTopicIterator.hasNext()) {
      MatrixSlice docSlice = docIterator.next();
      MatrixSlice topicSlice = docTopicIterator.next();
      int docId = docSlice.index();
      Vector document = docSlice.vector();
      Vector topicDist = topicSlice.vector();
      if (testFraction == 0 || docId % (1/testFraction) == 0) {
        trainSync(document, topicDist, false, 10);
        perplexity += readModel.perplexity(document, topicDist);
        matrixNorm += document.norm(1);
      }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.