Java 读取hdfs文件末尾多个字符

本代码思路来源于hadoop tail命令源码,tail只能读取偏移1024字符的数据,当需要的倒序读的字段大于1024的一种通用办法。有时候我们不需要完全读取文件,只需解析正序一部分,最后一部分数据。
代码如下

 public static void readHDFSFile(Path file) {
    try {
      Configuration configuration = new Configuration();
      configuration
          .set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());

      FileSystem fs = file.getFileSystem(configuration);
      FSDataInputStream fsin = fs.open(file);
      FileStatus fileStatus = fs.getFileStatus(file);
      InputStreamReader isr = new InputStreamReader(fsin, "utf-8");
      BufferedReader br = new BufferedReader(isr);
      long fsize = fileStatus.getLen();
      long endOffset = 14336;//自己设定倒读偏移量

      String containJobInitedStr = "org.apache.hadoop.mapreduce.jobhistory.JobInited";
      String containJobFinishedStr = "org.apache.hadoop.mapreduce.jobhistory.JobFinished";
      boolean hasReadInited = false;
      boolean isJobFinished = file.toString().contains("SUCCEEDED");
      String lstr;
      while ((lstr = br.readLine()) != null && !hasReadInited)) {
        if (!lstr.startsWith("{")) {
          continue;
        }
        if (lstr.contains(containJobInitedStr)) {//顺序读到某个标记处
          hasReadInited = true;
        }
        //do another thing
      }
      fsin.seek(fsize - endOffset);//重定位文件流
      String str;
      while ((str = br.readLine()) != null) {
        if (str.contains(containJobFinishedStr)) {//倒读获取需要的标记
          //do another thing
        }
      }
      fsin.close();
    } catch (Exception e) {
      e.printStackTrace();
    }

  }

猜你喜欢

转载自blog.csdn.net/banana1006034246/article/details/84987880