apache#5. 元数据管理

Author: Yifan122

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java

@@ -43,6 +43,8 @@ class FSDirMkdirOp {
 
   static HdfsFileStatus mkdirs(FSNamesystem fsn, String src,
       PermissionStatus permissions, boolean createParent) throws IOException {
+
+    //TODO FSDirectory目录树
     FSDirectory fsd = fsn.getFSDirectory();
     if(NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("DIR* NameSystem.mkdirs: " + src);
@@ -87,6 +89,7 @@ static HdfsFileStatus mkdirs(FSNamesystem fsn, String src,
           List<String> ancestors = nonExisting.subList(0, length - 1);
           // Ensure that the user can traversal the path by adding implicit
           // u+wx permission to all ancestor directories
+          // TODO 创建子目录
           existing = createChildrenDirectories(fsd, existing, ancestors,
               addImplicitUwx(permissions, permissions));
           if (existing == null) {
@@ -188,6 +191,7 @@ private static INodesInPath createSingleDirectory(FSDirectory fsd,
       INodesInPath existing, String localName, PermissionStatus perm)
       throws IOException {
     assert fsd.hasWriteLock();
+    // TODO 往内存中写元数据
     existing = unprotectedMkdir(fsd, fsd.allocateNewInodeId(), existing,
         localName.getBytes(Charsets.UTF_8), perm, null, now());
     if (existing == null) {
@@ -199,6 +203,7 @@ private static INodesInPath createSingleDirectory(FSDirectory fsd,
     // to match count of FilesDeleted metric.
     NameNode.getNameNodeMetrics().incrFilesCreated();
 
+    // TODO 重要，记录元数据
     String cur = existing.getPath();
     fsd.getEditLog().logMkDir(cur, newNode);
     if (NameNode.stateChangeLog.isDebugEnabled()) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java

@@ -414,17 +414,23 @@ synchronized List<FormatConfirmable> getFormatConfirmables() {
    * store yet.
    */
   void logEdit(final FSEditLogOp op) {
+
+    // 加锁
     synchronized (this) {
       assert isOpenForWrite() :
         "bad state: " + state;
 
       // wait if an automatic sync is scheduled
       waitIfAutoSyncScheduled();
-      
+
+      // TODO 步骤一： 获取当前的独一无二的事务ID
       long start = beginTransaction();
       op.setTransactionId(txid);
 
       try {
+        // TODO 步骤二：写入内存
+        // EditLogFileOutputStream
+        // QuorumOutputStream
         editLogStream.write(op);
       } catch (IOException ex) {
         // All journals failed, it is handled in logSync.
@@ -442,6 +448,7 @@ assert isOpenForWrite() :
     }
 
     // sync buffered edit log entries to persistent store
+    // TODO 持久化到磁盘
     logSync();
   }
 
@@ -485,6 +492,7 @@ private long beginTransaction() {
     //
     // record the transactionId when new data was written to the edits log
     //
+    // TODO txid本地副本 ThreadLocal
     TransactionId id = myTransactionId.get();
     id.txid = txid;
     return monotonicNow();
@@ -615,6 +623,7 @@ public void logSync() {
           sync = true;
 
           // swap buffers
+          // 交换buffer
           try {
             if (journalSet.isEmpty()) {
               throw new IOException("No journals available to flush");
@@ -632,7 +641,8 @@ public void logSync() {
             terminate(1, msg);
           }
         } finally {
-          // Prevent RuntimeException from blocking other log edit write 
+          // Prevent RuntimeException from blocking other log edit write
+          // TODO 恢复标识位，唤醒等待的线程
           doneWithAutoSyncScheduling();
         }
         //editLogStream may become null,
@@ -797,6 +807,8 @@ public void logUpdateBlocks(String path, INodeFile file, boolean toLogRpcIds) {
    */
   public void logMkDir(String path, INode newNode) {
     PermissionStatus permissions = newNode.getPermissionStatus();
+
+    // TODO 创建日志对象 【构建者模式】
     MkdirOp op = MkdirOp.getInstance(cache.get())
       .setInodeId(newNode.getId())
       .setPath(path)

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java

@@ -3863,6 +3863,7 @@ boolean mkdirs(String src, PermissionStatus permissions,
     try {
       checkOperation(OperationCategory.WRITE);
       checkNameNodeSafeMode("Cannot create directory " + src);
+      // TODO 创建目录
       auditStat = FSDirMkdirOp.mkdirs(this, src, permissions, createParent);
     } catch (AccessControlException e) {
       logAuditEvent(false, "mkdirs", src);

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java

@@ -48,6 +48,13 @@
  * We keep an in-memory representation of the file/block hierarchy.
  * This is a base INode class containing common fields for file and 
  * directory inodes.
+ *
+ * INode这个设计理念，其实是HDFS模仿Linux
+ *
+ * HDFS里面无论是目录还是文件，其实都是一个Inode
+ * 如果你是一个目录，那么你的类型就是INodeDirectory
+ * 如果你的类型是一个文件，那么就是INodeFile
+ *
  */
 @InterfaceAudience.Private
 public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java

@@ -974,6 +974,8 @@ public boolean mkdirs(String src, FsPermission masked, boolean createParent)
       throw new IOException("mkdirs: Pathname too long.  Limit " 
                             + MAX_PATH_LENGTH + " characters, " + MAX_PATH_DEPTH + " levels.");
     }
+
+    // TODO 调用FSNameSystem创建目录的方法
     return namesystem.mkdirs(src,
         new PermissionStatus(getRemoteUser().getShortUserName(),
             null, masked), createParent);