[SYCLomaitc][User define migration rules] Add "!include" syntax for user defined migration rule, re-org predefined IPEX and PyTorch rules (#2654)

Author: TejaX-Alaghari

clang/lib/DPCT/UserDefinedRules/UserDefinedRules.cpp

@@ -26,6 +26,7 @@ namespace clang {
 namespace dpct {
 std::vector<clang::tooling::UnifiedPath> MetaRuleObject::RuleFiles;
 std::vector<std::shared_ptr<MetaRuleObject>> MetaRules;
+llvm::DenseSet<llvm::StringRef> ProcessedYamlFiles;
 
 OutputBuilder::~OutputBuilder() {}
 
@@ -344,17 +345,147 @@ MetaRuleObject::PatternRewriter::PatternRewriter(
   Subrules = S;
 }
 
+// Read a YAML file recursively and substitute any "!include <filename>"
+// directive with the contents of the referenced file.
+std::unique_ptr<llvm::MemoryBuffer>
+readYAMLFile(const llvm::StringRef &RuleFilePath) {
+  // Check if the rule file has already been processed
+  // to avoid infinite recursion
+  if (!ProcessedYamlFiles.insert(RuleFilePath).second) {
+    return llvm::MemoryBuffer::getMemBufferCopy("");
+  }
+
+  // Load the rule file into a MemoryBuffer
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
+      llvm::MemoryBuffer::getFile(RuleFilePath);
+  if (!Buffer) {
+    llvm::errs() << "Error: failed to read " << RuleFilePath << ": "
+                 << Buffer.getError().message() << "\n";
+    clang::dpct::ShowStatus(MigrationErrorInvalidRuleFilePath);
+    dpctExit(MigrationErrorInvalidRuleFilePath);
+  }
+
+  // Get the directory path of the rule file.
+  llvm::SmallString<128> DirectoryPath(RuleFilePath);
+  llvm::sys::path::remove_filename(DirectoryPath);
+
+  // Iterate over the input line by line.
+  std::stringstream Output, IncRuleFilePath;
+  const std::string IncDirective = "!include";
+
+  size_t Idx = 0;
+  bool IncDirectiveFound = false;
+  bool SkipWhiteSpaces = false;
+  bool IsLineBeginning = true;
+
+  llvm::StringRef BuffContent = std::move(*Buffer)->getBuffer();
+  const auto BuffSize = BuffContent.size();
+  while (Idx < BuffSize) {
+    unsigned char Ch = BuffContent[Idx];
+
+    // Skip white spaces at the beginning of the line if it contains a directive
+    if (IsLineBeginning) {
+      auto i = Idx;
+      auto c = Ch;
+
+      // lookahead for "!" directive after white spaces
+      for (; i < BuffContent.size(); i++) {
+        c = BuffContent[i];
+
+        // Stop at new line or first non-white space character
+        if (c == '\n' || !std::isspace(c)) {
+          break;
+        }
+      }
+
+      // Check if the line starts with a directive
+      if (c == '!') {
+        // Move Idx to the beginning of directive
+        Idx = i;
+
+        // Check if the directive is "!include"
+        if (!IncDirectiveFound && Idx + IncDirective.length() <= BuffSize &&
+            BuffContent.substr(Idx, IncDirective.length()) == IncDirective) {
+          // Move Idx to the end of directive
+          Idx += IncDirective.length();
+          IncDirectiveFound = true;
+          SkipWhiteSpaces = true;
+        }
+
+        // Update current character
+        Ch = BuffContent[Idx];
+      }
+
+      IsLineBeginning = false;
+    }
+
+    // Skip return carriage character
+    if (Ch == '\r') {
+      Idx++;
+      continue;
+    }
+
+    // Process IncRuleFilePath at end of the line
+    if (Ch == '\n') {
+      if (IncDirectiveFound) {
+        auto IncRuleFilePathStr = IncRuleFilePath.str();
+
+        if (!IncRuleFilePathStr.empty()) {
+          // Find the absolute path for the included rule file path
+          llvm::SmallString<128> IncRuleFileAbsPath = DirectoryPath;
+          llvm::sys::path::append(IncRuleFileAbsPath, IncRuleFilePathStr);
+
+          // Recursively process the included file
+          if (llvm::sys::fs::exists(IncRuleFileAbsPath)) {
+            Output << readYAMLFile(IncRuleFileAbsPath.str())->getBuffer().str();
+          } else {
+            Output << readYAMLFile(IncRuleFilePathStr)->getBuffer().str();
+          }
+
+          // Clear the contents of include rule file path
+          IncRuleFilePath.str("");
+        }
+      }
+
+      // Reset include directive info for each new line
+      IncDirectiveFound = false;
+      SkipWhiteSpaces = false;
+      IsLineBeginning = true;
+    }
+
+    if (IncDirectiveFound) {
+      // Skip adding quotes to the include rule file path
+      if (Ch == '"' || Ch == '\'') {
+        Idx++;
+        // Flip white space skip flag at the boundaries of quotes
+        SkipWhiteSpaces = !SkipWhiteSpaces;
+        continue;
+      }
+
+      // Skip white space characters
+      if (SkipWhiteSpaces && std::isspace(Ch)) {
+        Idx++;
+        continue;
+      }
+
+      // Append the character to the include rule file path for !include line
+      IncRuleFilePath << Ch;
+    } else {
+      // Append the character to the output buffer
+      Output << Ch;
+    }
+
+    Idx++;
+  }
+
+  return llvm::MemoryBuffer::getMemBufferCopy(Output.str(), RuleFilePath);
+}
+
 void importRules(std::vector<clang::tooling::UnifiedPath> &RuleFiles) {
   for (auto &RuleFile : RuleFiles) {
     // open the yaml file
     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> Buffer =
-        llvm::MemoryBuffer::getFile(RuleFile.getCanonicalPath());
-    if (!Buffer) {
-      llvm::errs() << "Error: failed to read " << RuleFile << ": "
-                   << Buffer.getError().message() << "\n";
-      clang::dpct::ShowStatus(MigrationErrorInvalidRuleFilePath);
-      dpctExit(MigrationErrorInvalidRuleFilePath);
-    }
+        readYAMLFile(RuleFile.getCanonicalPath());
 
     // load rules
     std::vector<std::shared_ptr<MetaRuleObject>> CurrentRules;

clang/test/dpct/python_migration/case_001/include_rule_file/expected.py

@@ -0,0 +1,10 @@
+import a_repl
+import b_repl
+
+a_repl()
+b_repl()
+b1_repl()
+b2_repl()
+b3_repl()
+b4_repl()
+b5_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/include_rule_file.cpp

@@ -0,0 +1,13 @@
+// RUN: rm -rf %T && mkdir -p %T
+// RUN: cd %T
+// RUN: cp %S/input.py ./
+// RUN: cp %S/expected.py ./
+// RUN: cp -r %S/rules ./
+// RUN: echo "!include %T/rules/b/b5.yaml" >> %T/rules/python_build_script_migration_rule_inc.yaml
+
+// RUN: dpct -in-root ./ --migrate-build-script-only --migrate-build-script=Python --rule-file=%T/rules/python_build_script_migration_rule_inc.yaml input.py
+// RUN: echo "begin" > %T/diff.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/dpct_output/input.py >> %T/diff.txt
+// RUN: echo "end" >> %T/diff.txt
+// CHECK: begin
+// CHECK-NEXT: end

clang/test/dpct/python_migration/case_001/include_rule_file/input.py

@@ -0,0 +1,10 @@
+import a
+import b
+
+a()
+b()
+b1()
+b2()
+b3()
+b4()
+b5()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/a.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_a
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_a
+  In: a()
+  Out: a_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b 4.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b4
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b4
+  In: b4()
+  Out: b4_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b
+  In: b()
+  Out: b_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b1.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b1
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b1
+  In: b1()
+  Out: b1_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b2.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b2
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b2
+  In: b2()
+  Out: b2_repl()

clang/test/dpct/python_migration/case_001/include_rule_file/rules/b/b3.yaml

@@ -0,0 +1,7 @@
+- Rule: rule_call_b3
+  Kind: PythonRule
+  Priority: Fallback
+  MatchMode: Partial
+  PythonSyntax: call_b3
+  In: b3()
+  Out: b3_repl()