Adding Jira DataProcessor

src/components/admin/AdminApp.py

@@ -49,3 +49,21 @@ def post(session, _id: str):
   logger.debug(f"Entering {Routes.ImportHolidays} with args {debug_session(session)}, {_id=}")
   instance = InstanceManager.get(session, _id)
   return instance.import_holidays()
+
+@rt(Routes.ConfigureJira)
+def get(session, _id: str, boundaries: str):
+  logger.debug(f"Entering {Routes.ConfigureJira} - GET with args {debug_session(session)}, {_id=}, {boundaries=}")
+  instance = InstanceManager.get(session, _id)
+  return instance.show_configure_jira(json.loads(boundaries) if boundaries else None)
+
+@rt(Routes.ConfigureJira)
+def post(session, _id: str, args: dict):
+  logger.debug(f"Entering {Routes.ConfigureJira} - POST with args {debug_session(session)}, {_id=}, {args=}")
+  instance = InstanceManager.get(session, _id)
+  return instance.update_jira_settings(args)
+
+@rt(Routes.ConfigureJiraCancel)
+def post(session, _id: str):
+  logger.debug(f"Entering {Routes.ConfigureJiraCancel} with args {debug_session(session)}, {_id=}")
+  instance = InstanceManager.get(session, _id)
+  return instance.cancel_jira_settings()

src/components/admin/admin_db_manager.py

@@ -23,9 +23,16 @@ class AiBuddySettingsEntry:
     self.ollama_port = port
 
 
+@dataclass()
+class JiraSettingsEntry:
+  user_name: str = ""
+  api_token: str = ""
+
+
 @dataclass
 class AdminSettings:
   ai_buddy: AiBuddySettingsEntry = field(default_factory=AiBuddySettingsEntry)
+  jira: JiraSettingsEntry = field(default_factory=JiraSettingsEntry)
 
 
 class AdminDbManager:
@@ -37,3 +44,8 @@ class AdminDbManager:
                                           ADMIN_SETTINGS_ENTRY,
                                           AdminSettings,
                                           "ai_buddy")
+    self.jira = NestedSettingsManager(session,
+                                      settings_manager,
+                                      ADMIN_SETTINGS_ENTRY,
+                                      AdminSettings,
+                                      "jira")

src/components/admin/assets/icons.py

@@ -0,0 +1,10 @@
+from fastcore.basics import NotStr
+
+icon_jira = NotStr("""<svg name="jara" viewBox="0 0 48 48" xmlns="http://www.w3.org/2000/svg">
+  <defs>
+    <style>.a{fill:none;stroke:#000000;stroke-linecap:round;stroke-linejoin:round;}</style>
+  </defs>
+  <path class="a" d="M5.5,22.9722h0a8.7361,8.7361,0,0,0,8.7361,8.7361h2.0556v2.0556A8.7361,8.7361,0,0,0,25.0278,42.5h0V22.9722Z"/>
+  <path class="a" d="M14.2361,14.2361h0a8.7361,8.7361,0,0,0,8.7361,8.7361h2.0556v2.0556a8.7361,8.7361,0,0,0,8.7361,8.7361h0V14.2361Z"/>
+  <path class="a" d="M22.9722,5.5h0a8.7361,8.7361,0,0,0,8.7361,8.7361h2.0556v2.0556A8.7361,8.7361,0,0,0,42.5,25.0278h0V5.5Z"/>
+</svg>""")

src/components/admin/commands.py

@@ -39,6 +39,30 @@ class AdminCommandManager(BaseCommandManager):
         "hx-vals": f'js:{{"_id": "{self._id}", boundaries: getTabContentBoundaries("{self._owner.tabs_manager.get_id()}")}}',
     }
   
+  def show_configure_jira(self):
+    return {
+        "hx-get": f"{ROUTE_ROOT}{Routes.ConfigureJira}",
+        "hx-target": f"#{self._owner.tabs_manager.get_id()}",
+        "hx-swap": "outerHTML",
+        "hx-vals": f'js:{{"_id": "{self._id}", boundaries: getTabContentBoundaries("{self._owner.tabs_manager.get_id()}")}}',
+    }
+  
+  def save_configure_jira(self):
+    return {
+        "hx-post": f"{ROUTE_ROOT}{Routes.ConfigureJira}",
+        "hx-target": f"#{self._owner.tabs_manager.get_id()}",
+        "hx-swap": "outerHTML",
+        "hx-vals": f'js:{{"_id": "{self._id}"}}',
+        # The form adds the rest
+    }
+  
+  def cancel_configure_jira(self):
+    return {
+        "hx-post": f"{ROUTE_ROOT}{Routes.ConfigureJiraCancel}",
+        "hx-target": f"#{self._owner.tabs_manager.get_id()}",
+        "hx-swap": "outerHTML",
+        "hx-vals": f'js:{{"_id": "{self._id}"}}',
+    }
 
 class ImportHolidaysCommandManager(BaseCommandManager):
   def __init__(self, owner):

src/components/admin/components/Admin.py

@@ -4,10 +4,11 @@ from ai.mcp_client import MPC_CLIENTS_IDS
 from ai.mcp_tools import MCPServerTools
 from components.BaseComponent import BaseComponent
 from components.admin.admin_db_manager import AdminDbManager
+from components.admin.assets.icons import icon_jira
 from components.admin.commands import AdminCommandManager
 from components.admin.components.AdminForm import AdminFormItem, AdminFormType, AdminForm
 from components.admin.components.ImportHolidays import ImportHolidays
-from components.admin.constants import ADMIN_INSTANCE_ID, ADMIN_AI_BUDDY_INSTANCE_ID, ADMIN_IMPORT_HOLIDAYS_INSTANCE_ID
+from components.admin.constants import ADMIN_INSTANCE_ID, ADMIN_AI_BUDDY_INSTANCE_ID, ADMIN_JIRA_INSTANCE_ID
 from components.aibuddy.assets.icons import icon_brain_ok
 from components.hoildays.assets.icons import icon_holidays
 from components.tabs.components.MyTabs import MyTabs
@@ -59,6 +60,30 @@ class Admin(BaseComponent):
     
     return self._add_tab(ADMIN_AI_BUDDY_INSTANCE_ID, "Admin - Import Holidays", form)
   
+  def show_configure_jira(self, boundaries):
+    fields = [
+        AdminFormItem('user_name', "Email", "Email used to connect to JIRA.", AdminFormType.TEXT),
+        AdminFormItem("api_token", "API Key", "API Key to connect to JIRA.", AdminFormType.TEXT),
+    ]
+    hooks = {
+        "on_ok": self.commands.save_configure_jira(),
+        "on_cancel": self.commands.cancel_configure_jira(),
+        "ok_title": "Apply"
+    }
+    
+    form = InstanceManager.get(self._session,
+                               AdminForm.create_component_id(self._session, prefix=self._id),
+                               AdminForm,
+                               owner=self,
+                               title="Jira Configuration Page",
+                               obj=self.db.jira,
+                               form_fields=fields,
+                               hooks=hooks,
+                               key=ADMIN_JIRA_INSTANCE_ID,
+                               boundaries=boundaries
+                               )
+    return self._add_tab(ADMIN_JIRA_INSTANCE_ID, "Admin - Jira Configuration", form)
+  
   def update_ai_buddy_settings(self, values: dict):
     values = self.manage_lists(values)
     self.db.ai_buddy.update(values, ignore_missing=True)
@@ -69,6 +94,17 @@ class Admin(BaseComponent):
     self.tabs_manager.remove_tab(tab_id)
     return self.tabs_manager.render()
   
+  def update_jira_settings(self, values: dict):
+    values = self.manage_lists(values)
+    self.db.jira.update(values, ignore_missing=True)
+    return self.tabs_manager.render()
+
+  def cancel_jira_settings(self):
+    tab_id = self.tabs_manager.get_tab_id(ADMIN_JIRA_INSTANCE_ID)
+    self.tabs_manager.remove_tab(tab_id)
+    return self.tabs_manager.render()
+
+  
   def __ft__(self):
     return Div(
       Div(cls="divider"),
@@ -84,6 +120,11 @@ class Admin(BaseComponent):
         mk_ellipsis("holidays", cls="text-sm", **self.commands.show_import_holidays()),
         cls="flex p-0 min-h-0 truncate",
       ),
+      Div(
+        mk_icon(icon_jira, can_select=False),
+        mk_ellipsis("jira", cls="text-sm", **self.commands.show_configure_jira()),
+        cls="flex p-0 min-h-0 truncate",
+      ),
       #
       #   cls=""),
       # Script(f"bindAdmin('{self._id}')"),

src/components/admin/constants.py

@@ -1,6 +1,7 @@
 ADMIN_INSTANCE_ID = "__Admin__"
 ADMIN_AI_BUDDY_INSTANCE_ID = "__AdminAIBuddy__"
 ADMIN_IMPORT_HOLIDAYS_INSTANCE_ID = "__AdminImportHolidays__"
+ADMIN_JIRA_INSTANCE_ID = "__AdminJira__"
 ROUTE_ROOT = "/admin"
 ADMIN_SETTINGS_ENTRY = "Admin"
 
@@ -9,3 +10,5 @@ class Routes:
   AiBuddyCancel = "/ai-buddy-cancel"
   ImportHolidays = "/import-holidays"
   PasteHolidays = "/paste-holidays"
+  ConfigureJira = "/configure-jira"
+  ConfigureJiraCancel = "/configure-jira-cancel"

src/components/workflows/components/WorkflowPlayer.py

@@ -13,7 +13,7 @@ from components.workflows.db_management import WorkflowComponentRuntimeState, \
   WorkflowComponent, ComponentState
 from core.instance_manager import InstanceManager
 from core.utils import get_unique_id, make_safe_id
-from workflow.engine import WorkflowEngine, TableDataProducer, DefaultDataPresenter, DefaultDataFilter
+from workflow.engine import WorkflowEngine, TableDataProducer, DefaultDataPresenter, DefaultDataFilter, JiraDataProducer
 
 grid_settings = DataGridSettings(
   header_visible=True,
@@ -180,19 +180,25 @@ class WorkflowPlayer(BaseComponent):
     # first reorder the component, according to the connection definitions
     engine = WorkflowEngine()
     for component in sorted_components:
+      key = (component.type, component.properties["processor_name"])
       try:
-        if component.type == ProcessorTypes.Producer and component.properties["processor_name"] == "Repository":
+        if key == (ProcessorTypes.Producer, "Repository"):
           engine.add_processor(
             TableDataProducer(self._session,
                               self._settings_manager,
                               component.id,
                               component.properties["repository"],
                               component.properties["table"]))
-        
+        elif key == (ProcessorTypes.Producer, "Jira"):
-        elif component.type == ProcessorTypes.Filter and component.properties["processor_name"] == "Default":
+          engine.add_processor(
+            JiraDataProducer(self._session,
+                             self._settings_manager,
+                             component.id,
+                             'issues',
+                             component.properties["jira_jql"]))
+        elif key == (ProcessorTypes.Filter, "Default"):
           engine.add_processor(DefaultDataFilter(component.id, component.properties["filter"]))
-        
+        elif key == (ProcessorTypes.Presenter, "Default"):
-        elif component.type == ProcessorTypes.Presenter and component.properties["processor_name"] == "Default":
           engine.add_processor(DefaultDataPresenter(component.id, component.properties["columns"]))
         else:
           raise ValueError(

src/core/jira.py

@@ -0,0 +1,225 @@
+import json
+import logging
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+from core.Expando import Expando
+
+JIRA_ROOT = "https://altares.atlassian.net/rest/api/2"
+DEFAULT_HEADERS = {"Accept": "application/json"}
+
+logger = logging.getLogger("jql")
+
+
+class NotFound(Exception):
+  pass
+
+
+class Jira:
+  """Manage default operation to JIRA"""
+  
+  def __init__(self, user_name: str, api_token: str):
+    """
+    Prepare a connection to JIRA
+    The initialisation do not to anything,
+    It only stores the user_name and the api_token
+    Note that user_name and api_token is the recommended way to connect,
+        therefore, the only supported here
+    :param user_name:
+    :param api_token:
+    """
+    self.user_name = user_name
+    self.api_token = api_token
+    self.auth = HTTPBasicAuth(self.user_name, self.api_token)
+  
+  def issue(self, issue_id: str) -> Expando:
+    """
+    Retrieve an issue
+    :param issue_id:
+    :return:
+    """
+    url = f"{JIRA_ROOT}/issue/{issue_id}"
+    
+    response = requests.request(
+      "GET",
+      url,
+      headers=DEFAULT_HEADERS,
+      auth=self.auth
+    )
+    
+    return Expando(json.loads(response.text))
+  
+  def fields(self) -> list[Expando]:
+    """
+    Retrieve the list of all fields for an issue
+    :return:
+    """
+    url = f"{JIRA_ROOT}/field"
+    
+    response = requests.request(
+      "GET",
+      url,
+      headers=DEFAULT_HEADERS,
+      auth=self.auth
+    )
+    
+    as_dict = json.loads(response.text)
+    return [Expando(field) for field in as_dict]
+  
+  def jql(self, jql: str, fields="summary,status,assignee") -> list[Expando]:
+    """
+    Executes a JQL and returns the list of issues
+    :param jql:
+    :param fields: list of fields to retrieve
+    :return:
+    """
+    logger.debug(f"Processing jql '{jql}'")
+    url = f"{JIRA_ROOT}/search"
+    
+    headers = DEFAULT_HEADERS.copy()
+    headers["Content-Type"] = "application/json"
+    
+    payload = {
+        "fields": [f.strip() for f in fields.split(",")],
+        "fieldsByKeys": False,
+        "jql": jql,
+        "maxResults": 500,  # Does not seem to be used. It's always 100 !
+        "startAt": 0
+    }
+    
+    result = []
+    while True:
+      logger.debug(f"Request startAt '{payload['startAt']}'")
+      response = requests.request(
+        "POST",
+        url,
+        data=json.dumps(payload),
+        headers=headers,
+        auth=self.auth
+      )
+      
+      if response.status_code != 200:
+        raise Exception(self._format_error(response))
+      
+      as_dict = json.loads(response.text)
+      result += as_dict["issues"]
+      
+      if as_dict["startAt"] + as_dict["maxResults"] >= as_dict["total"]:
+        # We retrieve more than the total nuber of items
+        break
+      
+      payload["startAt"] += as_dict["maxResults"]
+    
+    return [Expando(issue) for issue in result]
+  
+  def extract(self, jql, mappings, updates=None) -> list[dict]:
+    """
+    Executes a jql and returns list of dict
+    The <code>issue</code> object, returned by the <ref>jql</ref> methods
+    contains all the fields for Jira. They are not all necessary
+    This method selects the required fields
+    :param jql:
+    :param mappings:
+    :param updates:  List of updates (lambda on issue) to perform
+    :return:
+    """
+    logger.debug(f"Processing extract using mapping {mappings}")
+    
+    def _get_field(mapping):
+      """Returns the meaningful jira field, for the mapping description path"""
+      fields = mapping.split(".")
+      return fields[1] if len(fields) > 1 and fields[0] == "fields" else fields[0]
+    
+    # retrieve the list of requested fields from what was asked in the mapping
+    jira_fields = [_get_field(mapping) for mapping in mappings]
+    as_string = ", ".join(jira_fields)
+    issues = self.jql(jql, as_string)
+    
+    for issue in issues:
+      # apply updates if needed
+      if updates:
+        for update in updates:
+          update(issue)
+      
+      row = {cvs_col: issue.get(jira_path) for jira_path, cvs_col in mappings.items() if cvs_col is not None}
+      yield row
+  
+  def get_versions(self, project_key):
+    """
+    Given a project name and a version name
+    returns fixVersion number in JIRA
+    :param project_key:
+    :param version_name:
+    :return:
+    """
+    
+    url = f"{JIRA_ROOT}/project/{project_key}/versions"
+    
+    response = requests.request(
+      "GET",
+      url,
+      headers=DEFAULT_HEADERS,
+      auth=self.auth
+    )
+    
+    if response.status_code != 200:
+      raise NotFound()
+    
+    as_list = json.loads(response.text)
+    return [Expando(version) for version in as_list]
+  
+  def get_version(self, project_key, version_name):
+    """
+    Given a project name and a version name
+    returns fixVersion number in JIRA
+    :param project_key:
+    :param version_name:
+    :return:
+    """
+    
+    for version in self.get_versions(project_key):
+      if version.name == version_name:
+        return version
+    
+    raise NotFound()
+  
+  def get_all_fields(self):
+    """
+    Helper function that returns the list of all field that can be requested in an issue
+    :return:
+    """
+    url = f"{JIRA_ROOT}/field"
+    response = requests.request(
+      "GET",
+      url,
+      headers=DEFAULT_HEADERS,
+      auth=self.auth
+    )
+    
+    as_dict = json.loads(response.text)
+    return [Expando(issue) for issue in as_dict]
+  
+  @staticmethod
+  def update_customer_refs(issue: Expando, bug_only=True, link_name=None):
+    issue["ticket_customer_refs"] = []
+    if bug_only and issue.fields.issuetype.name != "Bug":
+      return
+    
+    for issue_link in issue.fields.issuelinks:  # [i_link for i_link in issue.fields.issuelinks if i_link["type"]["name"] == "Relates"]:
+      if link_name and issue_link["type"]["name"] not in link_name:
+        continue
+      
+      direction = "inwardIssue" if "inwardIssue" in issue_link else "outwardIssue"
+      related_issue_key = issue_link[direction]["key"]
+      if related_issue_key.startswith("ITSUP"):
+        issue.ticket_customer_refs.append(related_issue_key)
+        continue
+  
+  @staticmethod
+  def _format_error(response):
+    if "errorMessages" in response.text:
+      error_messages = json.loads(response.text)["errorMessages"]
+    else:
+      error_messages = response.text
+    return f"Error {response.status_code} : {response.reason} : {error_messages}"

src/workflow/engine.py

@@ -2,7 +2,9 @@ import ast
 from abc import ABC, abstractmethod
 from typing import Any, Generator
 
+from components.admin.admin_db_manager import AdminDbManager
 from core.Expando import Expando
+from core.jira import Jira
 from core.utils import UnreferencedNamesVisitor
 from utils.Datahelper import DataHelper
 
@@ -87,6 +89,22 @@ class TableDataProducer(DataProducer):
     yield from DataHelper.get(self._session, self.settings_manager, self.repository_name, self.table_name, Expando)
 
 
+class JiraDataProducer(DataProducer):
+  """Base class for data producers that emit data from Jira."""
+  
+  def __init__(self, session, settings_manager, component_id, jira_object='issues', jira_query=''):
+    super().__init__(component_id)
+    self._session = session
+    self.settings_manager = settings_manager
+    self.jira_object = jira_object
+    self.jira_query = jira_query
+    self.db = AdminDbManager(session, settings_manager).jira
+  
+  def emit(self, data: Any = None) -> Generator[Any, None, None]:
+    jira = Jira(self.db.user_name, self.db.api_token)
+    yield from jira.jql(self.jira_query)
+
+
 class DefaultDataPresenter(DataPresenter):
   """Default data presenter that returns the input data unchanged."""
   
@@ -156,6 +174,7 @@ class WorkflowEngine:
       # Recursively process through remaining processors
       yield from self._process_single_item(processed_item, processor_index + 1)
 
+  
   def run(self) -> Generator[Any, None, None]:
     """
     Run the workflow pipeline and yield results one by one.
@@ -173,7 +192,7 @@ class WorkflowEngine:
       self.global_error = "First processor must be a DataProducer"
       raise ValueError(self.global_error)
     
-    for item in first_processor.emit():
+    for item in first_processor.process(None):
       yield from self._process_single_item(item, 1)
   
   def run_to_list(self) -> list[Any]: