Package com.alimama.mdrill.json

Examples of com.alimama.mdrill.json.JSONObject$Null


      Collection<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();

    for(int i=0;i<jsonStr.length();i++)
    {
      SolrInputDocument doc=new SolrInputDocument();
      JSONObject obj=jsonStr.getJSONObject(i);
      Iterator keys = obj.keys();
      while (keys.hasNext()) {
        String key = (String) keys.next();
        doc.addField(key, obj.get(key));
      }
      docs.add(doc);
    }
   
    return insert(projectName, docs,false, coresfortest,FlushType.buffer);
View Full Code Here


      GetPartions.Shards shard = GetPartions.getshard(part, req.partionsAll,coreinfo.cores, coreinfo.ms);
      HigoJoinParams[] joins=req.parseJoins(coreinfo,shard);
      ArrayList<String> fqList =req.parseFq(tblconfig, shard);
   
     
      JSONObject jsonObj = new JSONObject();

      long t2=System.currentTimeMillis();
     
      String rtn= request(part,tblconfig,coreinfo,req,callback, fqList, shard,joins,jsonObj);
      long t3=System.currentTimeMillis();
      long timetaken=t3-t1;
      jsonObj.put("___timetaken", timetaken+"="+(t2-t1)+"+"+(t3-t2));
      logParams=MdrillRequestLog.logRequest(projectName, callback, startStr, rowsStr,req.queryStr, dist, fl, groupby, sort, order,leftjoin);
      LOG.info("timetaken:"+(timetaken)+",logParams2:"+MdrillRequestLog.cutString(logParams));
      if(timetaken>1000l*100)
      {
        LOG.info("longrequest:"+(timetaken)+",logParams2:"+MdrillRequestLog.cutString(logParams)+"@"+MdrillRequestLog.cutString(rtn));
 
View Full Code Here

      JSONArray jsonArray=jsonObj.getJSONObject("data").getJSONArray("docs");
      ArrayList<JSONObject> results=new ArrayList<JSONObject>();
      boolean iscontains=true;
      for(int i=0;i<jsonArray.length();i++)
      {
        JSONObject obj=jsonArray.getJSONObject(i);
        if(!obj.has(req.sortType.sortRow))
        {
          iscontains=false;
        }
        results.add(obj);
      }
View Full Code Here

          jsonArray.put(fmt.format(new Date(d1.getTime()-i*1000l*3600*24)));
        }
      }
   
     
      JSONObject jsonObj = new JSONObject();
      JSONObject jo = new JSONObject();
      jo.put("min", minValue);
      jo.put("max", maxValue);
      jo.put("total", total);
      jo.put("list", jsonArray);
      jsonObj.put("data", jo);
      jsonObj.put("code", "1");
      jsonObj.put("message", "success");
      jsonObj.put("fcsize", fcsize);
      return callback + "(" + jsonObj.toString() + ")";
View Full Code Here

  private static long DELAYTIME=1000l*3600;
 
  public static String getJson(String name) throws JSONException
  {
    ArmoryInfo info=getInfo(name);
    JSONObject jsonObj = new JSONObject();
    jsonObj.put("code", "1");
    JSONObject jsonObj2 = new JSONObject();
    JSONArray jsonArray = new JSONArray();

    for(ArmoryItemInfo i:info.info)
    {
      jsonArray.put(i.tomap());
    }
    jsonObj2.put("docs", jsonArray);

    jsonObj.put("data", jsonObj2);


    return jsonObj.toString();
View Full Code Here

    return m_fpsql;
  }
 
  public static void addJsonList(String json,String user) throws Exception
  {
    JSONObject jsonObj = new JSONObject(json);
    JSONArray list=jsonObj.getJSONObject("data").getJSONArray("users");
    for(int i=0;i<list.length();i++)
    {
      JSONObject obj=list.getJSONObject(i);
      addjson(obj.toString(),user);
    }
  }
View Full Code Here

  {
    return getJson(null,false);
  }
  public static String getJson(String daystart,boolean extamsg) throws JSONException, SQLException
  {
    JSONObject jsonObj = new JSONObject();
    jsonObj.put("code", "1");
    MySqlConn m_fpsql = getConn();
   
    String strSql = "select userid,email,cname,role,permission,'-' as queryday,opuser,optime from users_json order by role desc,userid";
    if(daystart!=null&&!daystart.isEmpty())
    {
      strSql = " select userid,email,cname,role,permission,opuser,optime from users_json where (cname not in (select nick from query_analyser.day_user_pv  where queryday>'"+daystart+"')) order by role desc,userid";
    }
   
    if(extamsg)
    {
      strSql = " select a.userid as userid,a.email as email ,a.cname as cname,a.role as role,a.permission as permission,a.opuser as opuser,a.optime as optime, b.queryday as queryday " +
          "from ( " +
          ""+strSql+") " +
      "  a " +
      " left join (select nick,max(queryday) as queryday from query_analyser.day_user_pv group by nick ) b" +
      " on a.cname=b.nick order by role desc,queryday desc" ;
     
    }

    jsonObj.put("_exehql", strSql);

    Connection conn = m_fpsql.getConn();
    Statement stmt = conn.createStatement();
    try {
      ResultSet res = stmt.executeQuery(strSql);
      JSONArray userlist=new JSONArray();
      while (res.next()) {
        JSONObject item = new JSONObject();
        item.put("userid", String.valueOf(res.getString("userid")));
        item.put("email", String.valueOf(res.getString("email")));
        item.put("queryday", String.valueOf(res.getString("queryday")));

       
        item.put("cname", String.valueOf(res.getString("cname")));

        item.put("role", Integer.parseInt(res.getString("role")));
        item.put("permission", new JSONArray(res.getString("permission")));
        item.put("opuser", String.valueOf(res.getString("opuser")));
        item.put("optime", String.valueOf(res.getString("optime")));


        userlist.put(item);
       }
      m_fpsql.close();
      jsonObj.put("code", "1");
      jsonObj.put("message", "success");
      JSONObject dddd = new JSONObject();
      dddd.put("users", userlist);
      jsonObj.put("data", dddd);
    } catch (Exception e) {
      jsonObj.put("message", e.toString());
      jsonObj.put("code", "0");
    }finally{
View Full Code Here

    return jsonObj.toString();
  }
 
  public static void addjson(String json,String user) throws Exception
  {
    JSONObject jsonObj = new JSONObject(json);
    String userid=jsonObj.getString("userid");
    HashMap<String,String> val=new HashMap<String, String>();
    val.put("userid", jsonObj.getString("userid"));
    val.put("email", jsonObj.getString("email"));
    val.put("cname", jsonObj.getString("cname"));
    val.put("role", String.valueOf(jsonObj.getInt("role")));
    val.put("permission", jsonObj.getJSONArray("permission").toString());
    add(userid, val,user);
  }
View Full Code Here

  }
 
  public static void readJoinResult(String uuid, OutputStreamWriter outStream)
  throws SQLException, IOException, JSONException {
    final HashMap<String,String> tableInfo=getTableInfo(uuid);
    JSONObject jsonObj = new JSONObject();

    if(tableInfo==null||tableInfo.isEmpty())
    {
      jsonObj.put("code", "0");
      jsonObj.put("message""该表不存在");
      outStream.append(jsonObj.toString());
      return ;
    }
    if(tableInfo.get("status").equals("INDEXING"))
    {
      jsonObj.put("code", "0");
      jsonObj.put("message", "正在创建索引中请稍后");
      outStream.append(jsonObj.toString());
      return ;
    }
    Map stormconf = Utils.readStormConfig();
   
    Configuration conf=getConf(stormconf);
View Full Code Here

 
    private static final ExecutorService       EXECUTE  = ExecutorSerives.EXECUTE;

  public static String addTxt(final String tableName,final String store,final String callback) throws JSONException, SQLException, IOException
  {
    JSONObject jsonObj = new JSONObject();
    final HashMap<String,String> tableInfo=getTableInfo(tableName);
    if(tableInfo==null||tableInfo.isEmpty())
    {
      jsonObj.put("code", "0");
      jsonObj.put("message""该表不存在");
      if (callback != null && callback.length() > 0) {
        return callback + "(" + jsonObj.toString() + ")";
      } else {
        return jsonObj.toString();
      }
    }
    if(tableInfo.get("status").equals("INDEXING"))
    {
      jsonObj.put("code", "0");
      jsonObj.put("message", "正在创建索引中请稍后");
      if (callback != null && callback.length() > 0) {
        return callback + "(" + jsonObj.toString() + ")";
      } else {
        return jsonObj.toString();
      }
    }
    final Map stormconf = Utils.readStormConfig();
    final Configuration conf=getConf(stormconf);
    final FileSystem fs=FileSystem.get(conf);
    long size=HadoopUtil.duSize(fs, new Path(store));
    if(size>500l*1024*1024)
    {
      jsonObj.put("code", "0");
      jsonObj.put("message", "上传的文件不能超过500m");
      if (callback != null && callback.length() > 0) {
        return callback + "(" + jsonObj.toString() + ")";
      } else {
        return jsonObj.toString();
      }
    }

   
    TableJoin.updatePercent(tableName, "Stage-1 map = 0%,  reduce = 0%", "INDEXING";

    jsonObj.put("code", "1");

    EXECUTE.submit(new Runnable() {
      @Override
      public void run() {
        try{
        SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd");
        String day = fmt.format(new Date());
       
        if(!fs.exists(new Path(store)))
        {
          fs.mkdirs(new Path(store));
        }
       
       
        Path txtpath=new Path(tableInfo.get("txtStorePath"));
        if(!fs.exists(txtpath))
        {
          fs.mkdirs(txtpath);
        }
       
        for( FileStatus outpath:fs.listStatus(new Path(store)))
        {
          if(!outpath.isDir())
          {
            fs.rename(outpath.getPath(), new Path(txtpath,outpath.getPath().getName()+"_"+System.currentTimeMillis()));
          }
        }
        fs.delete(new Path(store),true);
       
        Path basepath=new Path(MdrillService.getBasePath());
        FileStatus[] tablelist=fs.listStatus(basepath);
        String solrHome=tablelist[0].getPath().toString();

        for(FileStatus tbl:tablelist){
          if(tbl.isDir()&&fs.exists(new Path(tbl.getPath(),"solr")))
          {
            solrHome=tbl.getPath().toString();
          }
        }
       
        HashSet<String> inputs=new HashSet<String>();
        inputs.add(txtpath.getName());
       

        TableJoin.updateKb(tableName, HadoopUtil.duSize(fs, txtpath));
                                 
        String strjoins="";
        if(tableInfo!=null&&tableInfo.get("joins")!=null)
        {
          for(String s:tableInfo.get("joins").split(","))
          {
            String[] cols=s.split(":");
            if(cols.length>=3)
            {
              strjoins=cols[2];
            }
          }
        }
       
        String index = (String) stormconf.get("higo.download.offline.store")+ "/" + day + "/tmp_" + java.util.UUID.randomUUID().toString();
        MakeIndex.make(fs, solrHome, conf, "txt", txtpath.getParent().toString(), inputs, "*", tableInfo.get("indexStorePath"), new Path(index), 1, tableInfo.get("splitString"), false, tableInfo.get("colsName"),new updateStatus() {

          @Override
          public void update(int statge, Job job) {
            try {
              TableJoin.LOG.info("update "+tableName+ ",stage:"+statge+",map:"+job.mapProgress()+",reduce:"+job.reduceProgress()+ ":INDEXING");
              String percent="Stage-"+statge+" map = "+(job.mapProgress()*100)+"%,  reduce = "+(job.reduceProgress()*100)+"%";
              TableJoin.updatePercent(tableName, percent, "INDEXING";
            } catch (Exception e) {
              TableJoin.LOG.error("updatePercent",e);
            }       
          }

          @Override
          public void finish() {
            try {
              TableJoin.LOG.info("update "+tableName+ ",INDEX");
              TableJoin.updatePercent(tableName, "Stage-2 map = 100%,  reduce = 100%", "INDEX";
            } catch (Exception e) {
              TableJoin.LOG.error("updatePercent",e);
            }
          }

          @Override
          public boolean dump(Job job) {
              Counters counters;
            try {
              counters = job.getCounters();
                    long wsize=counters.findCounter("higo", "dumpcount").getValue();
                    if(wsize>0)
                    {
                      try {
                  TableJoin.LOG.info("update "+tableName+ ",dump");
                  TableJoin.updatePercent(tableName, "Stage-2 map = 100%,  reduce = 100%", "DUMP";
                } catch (Exception e) {
                  TableJoin.LOG.error("updatePercent",e);
                }
                      return true;
                    }
            } catch (IOException e) {
              // TODO Auto-generated catch block
              e.printStackTrace();
            }
           
            return false;
           
          }
        },strjoins,40);
      }catch(Exception e){
        TableJoin.LOG.error("make index",e);
        try {
          TableJoin.updatePercent(tableName, "Stage-2 map = 100%,  reduce = 100%", "FAIL";
        } catch (Exception e2) {
          TableJoin.LOG.error("updatePercent",e2);
        }
      }
       
      }
    });
   
    if (callback != null && callback.length() > 0) {
      return callback + "(" + jsonObj.toString() + ")";
    } else {
      return jsonObj.toString();
    }
  }
View Full Code Here

TOP

Related Classes of com.alimama.mdrill.json.JSONObject$Null

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.