feat: improve gitloader component (#5351)

* feat: improve gitloader component

* [autofix.ci] apply automated fixes

* feat(git.py): refactor GitLoaderComponent to support asynchronous operations and improve temporary directory management

- Convert methods to async to enhance performance with file operations.
- Implement async context manager for handling temporary clone directories.
- Update binary file check and content filtering to be asynchronous.

* fix(git.py): enhance GitLoaderComponent with improved file filtering and binary check

- Refactor file filtering logic to utilize fnmatch for pattern matching.
- Introduce a new method to check for binary files based on null byte detection.
- Update content filtering to handle exceptions more gracefully.
- Modify temporary directory cleanup to use rmdir instead of remove for better directory management.
- Adjust load_documents method to utilize asyncio.to_thread for lazy loading of documents.

* refactor(git.py): enhance GitLoaderComponent with improved file filtering and binary check

- Refactor binary file check to be synchronous and handle exceptions more gracefully.
- Introduce new methods for checking file patterns and content patterns, allowing for more flexible filtering.
- Consolidate filtering logic into a single method for better maintainability.
- Update load_documents method to run lazy loading in a separate thread for improved performance.

* feat(tests): add unit tests for GitLoaderComponent file filtering and binary detection

- Introduced new test suite for GitLoaderComponent, including tests for binary file detection and file pattern matching.
- Implemented temporary file creation for testing various file types and permissions.
- Added tests for combined filtering functionality, ensuring robust handling of file and content patterns.

* refactor(git.py): improve pattern handling and content filtering in GitLoaderComponent

- Refactored pattern handling to use a more descriptive variable name `pattern_list` for clarity.
- Enhanced content filtering by ensuring proper encoding when reading file content.
- Updated regex validation to include a test string check for better error handling.
- Removed unnecessary comments to streamline the code and improve readability.

* fix: exception TypeError

TypeError: object async_generator can't be used in 'await' expression

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@langflow.org>
Co-authored-by: italojohnny <italojohnnydosanjos@gmail.com>

src/backend/tests/unit/components/git/__init__.py

@@ -0,0 +1 @@
+"""Git component tests."""

src/backend/tests/unit/components/git/test_git_component.py

@@ -0,0 +1,134 @@
+import tempfile
+from pathlib import Path
+
+import pytest
+from langflow.components.git import GitLoaderComponent
+
+
+@pytest.fixture
+def git_component():
+    return GitLoaderComponent()
+
+
+@pytest.fixture
+def test_files():
+    """Create temporary test files for filtering."""
+    with tempfile.TemporaryDirectory() as temp_dir:
+        # Create a Python file
+        python_file = Path(temp_dir) / "test.py"
+        python_file.write_text("import langchain\nclass TestComponent:\n    pass")
+
+        # Create a text file
+        text_file = Path(temp_dir) / "test.txt"
+        text_file.write_text("This is a test file")
+
+        # Create a binary file
+        binary_file = Path(temp_dir) / "test.bin"
+        binary_file.write_bytes(b"Binary\x00Content")
+
+        # Create a directory for permission tests
+        no_access_dir = Path(temp_dir) / "no_access"
+        no_access_dir.mkdir()
+        no_access_file = no_access_dir / "secret.txt"
+        no_access_file.write_text("secret")
+        no_access_file.chmod(0o000)  # Remove all permissions
+
+        yield temp_dir
+
+
+def test_is_binary(git_component, test_files):
+    """Test binary file detection."""
+    temp_dir = Path(test_files)
+
+    # Test regular files
+    assert not git_component.is_binary(temp_dir / "test.py")
+    assert not git_component.is_binary(temp_dir / "test.txt")
+    assert git_component.is_binary(temp_dir / "test.bin")
+
+    # Test error cases
+    assert git_component.is_binary(temp_dir / "nonexistent.txt")  # Non-existent file
+    assert git_component.is_binary(temp_dir / "no_access" / "secret.txt")  # No permission
+
+
+def test_check_file_patterns(git_component, test_files):
+    """Test file pattern matching."""
+    temp_dir = Path(test_files)
+
+    # Test single pattern
+    assert git_component.check_file_patterns(temp_dir / "test.py", "*.py")
+    assert not git_component.check_file_patterns(temp_dir / "test.txt", "*.py")
+
+    # Test exclusion pattern
+    assert not git_component.check_file_patterns(temp_dir / "test.py", "!*.py")
+
+    # Test multiple patterns
+    assert git_component.check_file_patterns(temp_dir / "test.py", "*.py,*.txt")
+    assert git_component.check_file_patterns(temp_dir / "test.txt", "*.py,*.txt")
+
+    # Test mixed include/exclude
+    assert not git_component.check_file_patterns(temp_dir / "test.py", "*.py,!test.py")
+    assert git_component.check_file_patterns(temp_dir / "other.py", "*.py,!test.py")
+
+    # Test empty pattern (should include all)
+    assert git_component.check_file_patterns(temp_dir / "test.py", "")
+    assert git_component.check_file_patterns(temp_dir / "test.txt", "  ")
+
+    # Test invalid pattern (should treat as literal string)
+    assert not git_component.check_file_patterns(temp_dir / "test.py", "[")
+
+
+def test_check_content_pattern(git_component, test_files):
+    """Test content pattern matching."""
+    temp_dir = Path(test_files)
+
+    # Test simple content match
+    assert git_component.check_content_pattern(temp_dir / "test.py", r"import langchain")
+    assert not git_component.check_content_pattern(temp_dir / "test.txt", r"import langchain")
+
+    # Test regex pattern
+    assert git_component.check_content_pattern(temp_dir / "test.py", r"class.*Component")
+
+    # Test binary file
+    assert not git_component.check_content_pattern(temp_dir / "test.bin", r"Binary")
+
+    # Test invalid regex patterns
+    assert not git_component.check_content_pattern(temp_dir / "test.py", r"[")  # Unclosed bracket
+    assert not git_component.check_content_pattern(temp_dir / "test.py", r"*")  # Invalid quantifier
+    assert not git_component.check_content_pattern(temp_dir / "test.py", r"(?<)")  # Invalid lookbehind
+    assert not git_component.check_content_pattern(temp_dir / "test.py", r"\1")  # Invalid backreference
+
+
+def test_combined_filter(git_component, test_files):
+    """Test the combined filter function."""
+    temp_dir = Path(test_files)
+
+    # Test with both patterns
+    filter_func = git_component.build_combined_filter(
+        file_filter_patterns="*.py", content_filter_pattern=r"class.*Component"
+    )
+    assert filter_func(str(temp_dir / "test.py"))
+    assert not filter_func(str(temp_dir / "test.txt"))
+    assert not filter_func(str(temp_dir / "test.bin"))
+
+    # Test with only file pattern
+    filter_func = git_component.build_combined_filter(file_filter_patterns="*.py")
+    assert filter_func(str(temp_dir / "test.py"))
+    assert not filter_func(str(temp_dir / "test.txt"))
+
+    # Test with only content pattern
+    filter_func = git_component.build_combined_filter(content_filter_pattern=r"class.*Component")
+    assert filter_func(str(temp_dir / "test.py"))
+    assert not filter_func(str(temp_dir / "test.txt"))
+
+    # Test with empty patterns
+    filter_func = git_component.build_combined_filter()
+    assert filter_func(str(temp_dir / "test.py"))
+    assert filter_func(str(temp_dir / "test.txt"))
+    assert not filter_func(str(temp_dir / "test.bin"))  # Binary files still excluded
+
+    # Test error cases
+    filter_func = git_component.build_combined_filter(
+        file_filter_patterns="*.py", content_filter_pattern=r"class.*Component"
+    )
+    assert not filter_func(str(temp_dir / "nonexistent.txt"))  # Non-existent file
+    assert not filter_func(str(temp_dir / "no_access" / "secret.txt"))  # No permission