Github processing and reporting

Author: oree-xx

scripts/2-process/github_process.py

@@ -83,7 +83,7 @@ def process_totals_by_code_license(args, count_data):
         tool = str(row.TOOL_IDENTIFIER)
         count = int(row.COUNT)
 
-        if tool == "Total_repositories":
+        if tool == "Total public repositories":
             continue
 
         if tool in [
@@ -107,6 +107,31 @@ def process_totals_by_code_license(args, count_data):
     data_to_csv(args, data, file_path)
 
 
+def process_totals_by_license(args, count_data):
+    """
+    Processing count data: totals by License
+    """
+    LOGGER.info(process_totals_by_license.__doc__.strip())
+    data = {}
+
+    for row in count_data.itertuples(index=False):
+        tool = str(row.TOOL_IDENTIFIER)
+        count = int(row.COUNT)
+
+        if tool == "Total public repositories":
+            continue
+
+        data[tool] = count
+
+    data = pd.DataFrame(data.items(), columns=["License", "Count"])
+    data.sort_values("Count", ascending=False, inplace=True)
+    data.reset_index(drop=True, inplace=True)
+    file_path = shared.path_join(
+        PATHS["data_phase"], "github_totals_by_license.csv"
+    )
+    data_to_csv(args, data, file_path)
+
+
 def process_totals_by_restriction(args, count_data):
     """
     Processing count data: totals by Approved for Free Cultural Works
@@ -119,6 +144,9 @@ def process_totals_by_restriction(args, count_data):
         tool = str(row.TOOL_IDENTIFIER)
         count = int(row.COUNT)
 
+        if tool == "Total public repositories":
+            continue
+
         if tool in ["BSD Zero Clause License", "CC0 1.0", "Unlicense"]:
             key = "Public domain"
         elif tool in ["MIT No Attribution", "CC BY 4.0"]:
@@ -138,6 +166,44 @@ def process_totals_by_restriction(args, count_data):
     data_to_csv(args, data, file_path)
 
 
+def process_totals_by_rights_reserved(args, count_data):
+    """
+    Processing count data: totals by Rights Reserved
+    """
+    LOGGER.info(process_totals_by_rights_reserved.__doc__.strip())
+    data = {
+        "Rights reserved": 0,
+        "No rights reserved": 0,
+    }
+    for row in count_data.itertuples(index=False):
+        tool = str(row.TOOL_IDENTIFIER)
+        count = int(row.COUNT)
+
+        if tool == "Total public repositories":
+            continue
+
+        if tool in [
+            "MIT No Attribution",
+            "BSD Zero Clause License",
+            "CC0 1.0",
+            "Unlicense",
+        ]:
+            key = "No rights reserved"
+        elif tool in ["CC BY 4.0", "CC-BY-SA 4.0"]:
+            key = "Rights reserved"
+        else:
+            continue
+
+        data[key] += count
+    data = pd.DataFrame(data.items(), columns=["Category", "Count"])
+    data.sort_values("Count", ascending=False, inplace=True)
+    data.reset_index(drop=True, inplace=True)
+    file_path = shared.path_join(
+        PATHS["data_phase"], "github_totals_by_rights_reserved.csv"
+    )
+    data_to_csv(args, data, file_path)
+
+
 # def load_quarter_data(quarter):
 #     """
 #     Load data for a specific quarter.
@@ -188,6 +254,8 @@ def main():
 
     file_count = shared.path_join(PATHS["data_1-fetch"], "github_1_count.csv")
     count_data = pd.read_csv(file_count, usecols=["TOOL_IDENTIFIER", "COUNT"])
+    process_totals_by_license(args, count_data)
+    process_totals_by_rights_reserved(args, count_data)
     process_totals_by_restriction(args, count_data)
     process_totals_by_code_license(args, count_data)
 

scripts/3-report/github_report.py

@@ -86,18 +86,51 @@ def load_data(args):
     return data
 
 
-def Plot_by_license_type(args):
+def github_intro(args):
     """
-    Create plots for the languages with highest usage of latest tools
+    Write Github introduction.
     """
-    LOGGER.info(plot_totals_by_code_license.__doc__.strip())
+    LOGGER.info(github_intro.__doc__.strip())
     file_path = shared.path_join(
         PATHS["data_1-fetch"],
         "github_1_count.csv",
     )
     LOGGER.info(f"data file: {file_path.replace(PATHS['repo'], '.')}")
     name_label = "TOOL_IDENTIFIER"
-    data_label = "COUNT"
+    data = pd.read_csv(file_path, index_col=name_label)
+    total_repositories = data.loc["Total public repositories", "COUNT"]
+    shared.update_readme(
+        args,
+        SECTION,
+        "Overview",
+        None,
+        None,
+        "Github data uses the `total_count` returned by"
+        " API for search queries of the SPDX IDENTIFIER URLS"
+        "\n"
+        f"**The results indicate that a total of {total_repositories}"
+        "repositories on GitHub use a mix of some rights reserved and"
+        "no rights reserved licenses which showcases the usage of"
+        "attribution based Creative Commons (CC) legal tool"
+        "and Public domain equivalent.**\n"
+        "/n"
+        "Thank you GitHub for providing public access to"
+        "repository metadata through its API.",
+    )
+
+
+def Plot_by_license_type(args):
+    """
+    Create plots for the languages with highest usage of latest tools
+    """
+    LOGGER.info(plot_totals_by_code_license.__doc__.strip())
+    file_path = shared.path_join(
+        PATHS["data_2-process"],
+        "github_totals_by_license.csv",
+    )
+    LOGGER.info(f"data file: {file_path.replace(PATHS['repo'], '.')}")
+    name_label = "License"
+    data_label = "Count"
     data = pd.read_csv(file_path, index_col=name_label)
 
     title = "Totals by license type"
@@ -166,7 +199,7 @@ def plot_totals_by_code_license(args):
         SECTION,
         title,
         image_path,
-        "Plots showing totals by code license.",
+        "Plots showing totals by code license vs content license.",
     )
 
 
@@ -184,7 +217,7 @@ def plot_totals_by_restriction(args):
     data_label = "Count"
     data = pd.read_csv(file_path, index_col=name_label)
 
-    title = "Approved for Free Cultural Works"
+    title = "Totals by restriction"
     plt = plot.combined_plot(
         args=args,
         data=data,
@@ -211,13 +244,56 @@ def plot_totals_by_restriction(args):
     )
 
 
+def plot_totals_by_rights_reserved(args):
+    """
+    Create plots for the languages with highest usage of latest tools
+    """
+    LOGGER.info(plot_totals_by_rights_reserved.__doc__.strip())
+    file_path = shared.path_join(
+        PATHS["data_2-process"],
+        "github_totals_by_rights_reserved.csv",
+    )
+    LOGGER.info(f"data file: {file_path.replace(PATHS['repo'], '.')}")
+    name_label = "Category"
+    data_label = "Count"
+    data = pd.read_csv(file_path, index_col=name_label)
+
+    title = "Totals by Rights Reserved vs No Rights Reserved"
+    plt = plot.combined_plot(
+        args=args,
+        data=data,
+        title=title,
+        name_label=name_label,
+        data_label=data_label,
+    )
+
+    image_path = shared.path_join(
+        PATHS["data_phase"], "github_rights_reserved.png"
+    )
+    LOGGER.info(f"image file: {image_path.replace(PATHS['repo'], '.')}")
+    if args.enable_save:
+        # Create the directory if it does not exist
+        os.makedirs(PATHS["data_phase"], exist_ok=True)
+        plt.savefig(image_path)
+
+    shared.update_readme(
+        args,
+        SECTION,
+        title,
+        image_path,
+        "Plots showing totals by rights reserved vs No rights reserved.",
+    )
+
+
 def main():
     args = parse_arguments()
     shared.paths_log(LOGGER, PATHS)
     shared.git_fetch_and_merge(args, PATHS["repo"])
+    github_intro(args)
     plot_totals_by_restriction(args)
     plot_totals_by_code_license(args)
     Plot_by_license_type(args)
+    plot_totals_by_rights_reserved(args)
 
     # Add and commit changes
     args = shared.git_add_and_commit(