传统的解析JSON是要完全构建好一个完整的JSON文本内容然后通过JSON框架去解析反序列化,但是当JSON内容非常长,比如,一个JSON列表中存储有N个元组,而且非常长有上万个。这样在传输JSON文本内容的时候如何实现流式传输呢?方法当然是有的,在Solr的Xport代码中有就有这样一段代码,类名是:JSONTupleStream。
import java.io.IOException; import java.io.Reader; import java.util.List; import java.util.Map; import org.apache.solr.client.solrj.io.stream.SolrStream; import org.noggit.JSONParser; import org.noggit.ObjectBuilder; public class JSONTupleStream { private List<String> path; // future... for more general stream handling private Reader reader; private JSONParser parser; private boolean atDocs; public JSONTupleStream(Reader reader) { this.reader = reader; this.parser = new JSONParser(reader); } /** returns the next Tuple or null */ public Map<String,Object> next() throws IOException { if (!atDocs) { boolean found = advanceToDocs(); atDocs = true; if (!found) return null; } // advance past ARRAY_START (in the case that we just advanced to docs, or OBJECT_END left over from the last call. int event = parser.nextEvent(); if (event == JSONParser.ARRAY_END) return null; Object o = ObjectBuilder.getVal(parser); // right now, getVal will leave the last event read as OBJECT_END return (Map<String,Object>)o; } public void close() throws IOException { reader.close(); } private void expect(int parserEventType) throws IOException { int event = parser.nextEvent(); if (event != parserEventType) { throw new IOException("JSONTupleStream: expected " + JSONParser.getEventString(parserEventType) + " but got " + JSONParser.getEventString(event) ); } } private void expect(String mapKey) { } private boolean advanceToMapKey(String key, boolean deepSearch) throws IOException { for (;;) { int event = parser.nextEvent(); switch (event) { case JSONParser.STRING: if (key != null) { String val = parser.getString(); if (key.equals(val)) { return true; } else if("error".equals(val)) { handleError(); } } break; case JSONParser.OBJECT_END: return false; case JSONParser.OBJECT_START: if (deepSearch) { boolean found = advanceToMapKey(key, true); if (found) { return true; } } else { advanceToMapKey(null, false); } break; case JSONParser.ARRAY_START: skipArray(key, deepSearch); break; } } } private void handleError() throws IOException { for (;;) { int event = parser.nextEvent(); if(event == JSONParser.STRING) { String val = parser.getString(); if("msg".equals(val)) { event = parser.nextEvent(); if(event == JSONParser.STRING) { String msg = parser.getString(); if(msg != null) { throw new SolrStream.HandledException(msg); } } } } else if (event == JSONParser.OBJECT_END) { throw new IOException(""); } } } private void skipArray(String key, boolean deepSearch) throws IOException { for (;;) { int event = parser.nextEvent(); switch (event) { case JSONParser.OBJECT_START: advanceToMapKey(key, deepSearch); break; case JSONParser.ARRAY_START: skipArray(key, deepSearch); break; case JSONParser.ARRAY_END: return; } } } private boolean advanceToDocs() throws IOException { expect(JSONParser.OBJECT_START); advanceToMapKey("numFound", true); expect(JSONParser.LONG); // int event = parser.nextEvent(); //if (event == JSONParser.ARRAY_END) return null; System.out.println( ObjectBuilder.getVal(parser)); boolean found = advanceToMapKey("docs", true); expect(JSONParser.ARRAY_START); return found; } }
再写一段测脚本:
import java.io.StringReader; import java.util.Map; import com.dfire.tis.solrextend.core.JSONTupleStream; import junit.framework.TestCase; public class TestTupleStreamParser extends TestCase { public void test() throws Exception { StringReader reader = new StringReader("{\"responseHeader\": {\"status\": 0}, \"response\":{\"numFound\":" + 100 + ", \"docs\":[{name:'baisui'},{name:'xxxx'}]}"); JSONTupleStream tupleStream = new JSONTupleStream(reader); while (true) { Map<String, Object> tuple = tupleStream.next(); if (tuple != null) { System.out.println(tuple.get("name")); } else { break; } } } }
这样就可以做到,发送端不停得发送,客户端可以做到不断的去消费,这样做的好处是,当服务端客户端传输大量结构化数据的时候,可以避免首先要先开辟一个很大的内存空间,这样可以有效防止对内存溢出,同时,也可以有效提高系统吞吐率。