diff --git a/.gitignore b/.gitignore index 095449ce..b1674cf6 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,8 @@ build/ # Virtual environments *venv*/ **/*venv*/ + +# learning files +learnings/ +logging_docs/ +logging_demo/ \ No newline at end of file diff --git a/LOGGING.md b/LOGGING.md new file mode 100644 index 00000000..ac08bbeb --- /dev/null +++ b/LOGGING.md @@ -0,0 +1,828 @@ +# Logging Guide for mssql-python + +This guide explains how to use the logging system in mssql-python for comprehensive diagnostics and troubleshooting. + +## Table of Contents + +- [Quick Start](#quick-start) +- [Philosophy](#philosophy) +- [Basic Usage](#basic-usage) +- [Log Output Examples](#log-output-examples) +- [Advanced Features](#advanced-features) +- [API Reference](#api-reference) +- [Extensibility](#extensibility) + +## Quick Start + +### Minimal Usage (Recommended) + +```python +import mssql_python + +# Enable logging - shows EVERYTHING (one line) +mssql_python.setup_logging() + +# Use the driver - all operations are now logged +conn = mssql_python.connect("Server=localhost;Database=test") +# Check the log file: ./mssql_python_logs/mssql_python_trace_*.log +``` + +### With Output Control + +```python +import mssql_python + +# Enable logging (default: file only) +mssql_python.setup_logging() + +# Output to stdout instead of file +mssql_python.setup_logging(output='stdout') + +# Output to both file and stdout +mssql_python.setup_logging(output='both') + +# Custom log file path +mssql_python.setup_logging(log_file_path="/var/log/myapp.log") +``` + +## Philosophy + +**Simple and Purposeful:** +- **One Level**: All logs are DEBUG level - no categorization needed +- **All or Nothing**: When you enable logging, you see EVERYTHING (SQL, parameters, internal operations) +- **Troubleshooting Focus**: Turn on logging when something is broken, turn it off otherwise +- **⚠️ Performance Warning**: Logging has overhead - DO NOT enable in production without reason + +**Why No Multiple Levels?** +- If you need logging, you need to see what's broken - partial information doesn't help +- Simplifies the API and mental model +- Future enhancement: Universal profiler for performance analysis (separate from logging) + +**When to Enable Logging:** +- ✅ Debugging connection issues +- ✅ Troubleshooting query execution problems +- ✅ Investigating unexpected behavior +- ✅ Reproducing customer issues +- ❌ Evaluating query performance (use profiler instead - coming soon) +- ❌ Production monitoring (use proper monitoring tools) +- ❌ "Just in case" logging (adds unnecessary overhead) + +## Basic Usage + +### Default - File Logging + +```python +import mssql_python + +# Enable logging (logs to file by default) +mssql_python.setup_logging() + +# Use the library - logs will appear in file +conn = mssql_python.connect(server='localhost', database='testdb') +cursor = conn.cursor() +cursor.execute("SELECT * FROM users") + +# Access logger for file path (advanced) +from mssql_python.logging import logger +print(f"Logs written to: {logger.log_file}") +``` + +### Console Logging + +```python +import mssql_python + +# Enable logging to stdout +mssql_python.setup_logging(output='stdout') + +# Now use the library - logs will appear in console +conn = mssql_python.connect(server='localhost', database='testdb') +cursor = conn.cursor() +cursor.execute("SELECT * FROM users") +``` + +### Both File and Console + +```python +import mssql_python + +# Enable logging to both file and stdout +mssql_python.setup_logging(output='both') + +# Logs appear in both console and file +conn = mssql_python.connect(server='localhost', database='testdb') +``` + +### Custom Log File Path + +```python +import mssql_python + +# Specify custom log file path +mssql_python.setup_logging(log_file_path="/var/log/myapp/mssql.log") + +# Or with both file and stdout +mssql_python.setup_logging(output='both', log_file_path="/tmp/debug.log") + +conn = mssql_python.connect(server='localhost', database='testdb') + +# Check log file location +from mssql_python.logging import logger +print(f"Logging to: {logger.log_file}") +# Output: Logging to: /var/log/myapp/mssql.log +``` + +## Output Destinations + +### File Only (Default) + +```python +import mssql_python + +# File logging is enabled by default +mssql_python.setup_logging() + +# Files are automatically rotated at 512MB, keeps 5 backups +# File location: ./mssql_python_logs/mssql_python_trace_YYYYMMDDHHMMSS_PID.log +# (mssql_python_logs folder is created automatically if it doesn't exist) + +conn = mssql_python.connect(server='localhost', database='testdb') + +from mssql_python.logging import logger +print(f"Logging to: {logger.log_file}") +``` + +### Stdout Only + +```python +import mssql_python + +# Log to stdout only (useful for CI/CD, Docker containers) +mssql_python.setup_logging(output='stdout') + +conn = mssql_python.connect(server='localhost', database='testdb') +# Logs appear in console, no file created +``` + +### Both File and Stdout + +```python +import mssql_python + +# Log to both destinations (useful for development) +mssql_python.setup_logging(output='both') + +conn = mssql_python.connect(server='localhost', database='testdb') +# Logs appear in both console and file +``` + +## Log Output Examples + +### Standard Output + +When logging is enabled, you see EVERYTHING - SQL statements, parameters, internal operations. + +**File Header:** +``` +# MSSQL-Python Driver Log | Script: main.py | PID: 12345 | Log Level: DEBUG | Python: 3.13.7 | Start: 2025-11-06 10:30:15 +Timestamp, ThreadID, Level, Location, Source, Message +``` + +**Sample Entries:** +``` +2025-11-06 10:30:15.100, 8581947520, DEBUG, connection.py:156, Python, Allocating environment handle +2025-11-06 10:30:15.101, 8581947520, DEBUG, connection.cpp:22, DDBC, Allocating ODBC environment handle +2025-11-06 10:30:15.123, 8581947520, DEBUG, connection.py:42, Python, Connecting to server: localhost +2025-11-06 10:30:15.456, 8581947520, DEBUG, cursor.py:28, Python, Executing query: SELECT * FROM users WHERE id = ? +2025-11-06 10:30:15.457, 8581947520, DEBUG, cursor.py:89, Python, Query parameters: [42] +2025-11-06 10:30:15.789, 8581947520, DEBUG, cursor.py:145, Python, Fetched 1 row +2025-11-06 10:30:15.790, 8581947520, DEBUG, cursor.py:201, Python, Row buffer allocated +``` + +**Log Format:** +- **Timestamp**: Date and time with milliseconds +- **ThreadID**: OS native thread ID (matches debugger thread IDs) +- **Level**: DEBUG, INFO, WARNING, ERROR +- **Location**: filename:line_number +- **Source**: Python or DDBC (C++ layer) +- **Message**: The actual log message + +**What You'll See:** +- ✅ Connection establishment and configuration +- ✅ SQL query text +- ✅ Query parameters (with PII sanitization) +- ✅ Result set information +- ✅ Internal ODBC operations +- ✅ Memory allocations and handle management +- ✅ Transaction state changes +- ✅ Everything the driver does + +## Advanced Features + +### Password Sanitization + +Sensitive data like passwords and access tokens are automatically sanitized in logs: + +```python +conn = mssql_python.connect( + server='localhost', + database='testdb', + username='admin', + password='MySecretPass123!' +) + +# Log output shows: +# Connection string: Server=localhost;Database=testdb;UID=admin;PWD=***REDACTED*** +``` + +Keywords automatically sanitized: +- `password`, `pwd`, `passwd` +- `access_token`, `accesstoken` +- `secret`, `api_key`, `apikey` +- `token`, `auth`, `authentication` + +### Thread Tracking + +Each log entry includes the **OS native thread ID** for tracking operations in multi-threaded applications: + +**Thread ID Benefits:** +- **Debugger Compatible**: Thread IDs match those shown in debuggers (Visual Studio, gdb, lldb) +- **OS Native**: Same thread ID visible in system monitoring tools +- **Multi-threaded Tracking**: Easily identify which thread performed which operations +- **Performance Analysis**: Correlate logs with profiler/debugger thread views + +**Example:** +```python +import mssql_python +import threading + +# Enable logging +mssql_python.setup_logging() + +conn = mssql_python.connect("Server=localhost;Database=test") +cursor = conn.cursor() +cursor.execute("SELECT * FROM users") + +# Log output shows (CSV format): +# 2025-11-06 10:30:15.100, 8581947520, DEBUG, connection.py:42, Python, Connection established +# 2025-11-06 10:30:15.102, 8581947520, DEBUG, cursor.py:15, Python, Cursor created +# 2025-11-06 10:30:15.103, 8581947520, DEBUG, cursor.py:28, Python, Executing query: SELECT * FROM users + +# Different thread/connection (note different ThreadID): +# 2025-11-06 10:30:15.200, 8582001664, DEBUG, connection.py:42, Python, Connection established +``` + +**Why Thread IDs Matter:** +- **Multi-threading**: Distinguish logs from different threads writing to the same file +- **Connection pools**: Track which thread is handling which connection +- **Debugging**: Filter logs by thread ID using text tools (grep, awk, etc.) +- **Performance analysis**: Measure duration of specific operations per thread +- **Debugger Correlation**: Thread ID matches debugger views for easy debugging + +### Using mssql-python's Logger in Your Application + +You can access the same logger used by mssql-python in your application code: + +```python +import mssql_python +from mssql_python.logging import driver_logger + +# Enable logging first +mssql_python.setup_logging() + +# Now use driver_logger in your application +driver_logger.debug("[App] Starting data processing") +driver_logger.info("[App] Processing complete") +driver_logger.warning("[App] Resource usage high") +driver_logger.error("[App] Failed to process record") + +# Your logs will appear in the same file as driver logs, +# with the same format and thread tracking +``` + +**Benefits:** +- Unified logging - all logs in one place +- Same format and structure as driver logs +- Automatic thread ID tracking +- No need to configure separate loggers + +### Importing Logs as CSV (Optional) + +Log files use comma-separated format and can be imported into spreadsheet tools: + +```python +import pandas as pd + +# Import log file (skip header lines starting with #) +df = pd.read_csv('mssql_python_logs/mssql_python_trace_20251106103015_12345.log', + comment='#') + +# Filter by thread, analyze queries, etc. +thread_logs = df[df['ThreadID'] == 8581947520] +``` + +### Programmatic Log Access (Advanced) + +```python +import mssql_python +from mssql_python.logging import logger +import logging as py_logging + +# Add custom handler to process logs programmatically +class MyLogHandler(py_logging.Handler): + def emit(self, record): + # Process log record + print(f"Custom handler: {record.getMessage()}") + + # Access trace ID + trace_id = getattr(record, 'trace_id', None) + if trace_id: + print(f" Trace ID: {trace_id}") + +handler = MyLogHandler() +logger.addHandler(handler) + +# Now enable logging +mssql_python.setup_logging() +``` + +## API Reference + +### Primary Function + +**`mssql_python.setup_logging(output: str = 'file', log_file_path: str = None) -> None`** + +Enable comprehensive DEBUG logging for troubleshooting. + +**Parameters:** +- `output` (str, optional): Where to send logs. Options: `'file'` (default), `'stdout'`, `'both'` +- `log_file_path` (str, optional): Custom log file path. Must have extension: `.txt`, `.log`, or `.csv`. If not specified, auto-generates path in `./mssql_python_logs/` + +**Raises:** +- `ValueError`: If `log_file_path` has an invalid extension (only `.txt`, `.log`, `.csv` are allowed) + +**Examples:** + +```python +import mssql_python + +# Basic usage - file logging (default, auto-generated path) +mssql_python.setup_logging() + +# Output to stdout only +mssql_python.setup_logging(output='stdout') + +# Output to both file and stdout +mssql_python.setup_logging(output='both') + +# Custom log file path (must use .txt, .log, or .csv extension) +mssql_python.setup_logging(log_file_path="/var/log/myapp.log") +mssql_python.setup_logging(log_file_path="/tmp/debug.txt") +mssql_python.setup_logging(log_file_path="/tmp/data.csv") + +# Custom path with both outputs +mssql_python.setup_logging(output='both', log_file_path="/tmp/debug.log") + +# Invalid extensions will raise ValueError +try: + mssql_python.setup_logging(log_file_path="/tmp/debug.json") # ✗ Error +except ValueError as e: + print(e) # "Invalid log file extension '.json'. Allowed extensions: .csv, .log, .txt" +``` + +### Advanced - Using driver_logger in Your Code + +Access the same logger used by mssql-python in your application: + +```python +from mssql_python.logging import driver_logger +import mssql_python + +# Enable logging +mssql_python.setup_logging() + +# Use driver_logger in your application +driver_logger.debug("[App] Starting data processing") +driver_logger.info("[App] Processing complete") +driver_logger.warning("[App] Resource usage high") +driver_logger.error("[App] Failed to process record") + +# Your logs appear in the same file with same format +``` + +### Advanced - Logger Instance + +For advanced use cases, you can access the logger instance directly: + +```python +from mssql_python.logging import logger + +# Get log file path +print(f"Logging to: {logger.log_file}") + +# Add custom handlers (for integration) +import logging as py_logging +custom_handler = py_logging.StreamHandler() +logger.addHandler(custom_handler) + +# Direct logging calls (if needed) +logger.debug("Custom debug message") +``` + +## Extensibility + +### Pattern 1: Use Driver Logger Across Your Application + +If you want to use the driver's logger for your own application logging: + +```python +import mssql_python +from mssql_python.logging import logger + +# Enable driver logging +mssql_python.setup_logging(output='stdout') + +# Use the logger in your application +class MyApp: + def __init__(self): + logger.debug("Application starting") + self.db = self._connect_db() + logger.debug("Application ready") + + def _connect_db(self): + logger.debug("Connecting to database") + conn = mssql_python.connect("Server=localhost;Database=test") + logger.debug("Database connected successfully") + return conn + + def process_data(self): + logger.debug("Processing data") + cursor = self.db.cursor() + cursor.execute("SELECT COUNT(*) FROM users") + count = cursor.fetchone()[0] + logger.debug(f"Processed {count} users") + return count + +if __name__ == '__main__': + app = MyApp() + result = app.process_data() +``` + +**Output shows unified logging:** +``` +2025-11-03 10:15:22 - mssql_python - DEBUG - Application starting +2025-11-03 10:15:22 - mssql_python - DEBUG - Connecting to database +2025-11-03 10:15:22 - mssql_python - DEBUG - [Python] Initializing connection +2025-11-03 10:15:22 - mssql_python - DEBUG - Database connected successfully +2025-11-03 10:15:22 - mssql_python - DEBUG - Application ready +2025-11-03 10:15:22 - mssql_python - DEBUG - Processing data +2025-11-03 10:15:22 - mssql_python - DEBUG - [Python] Executing query +2025-11-03 10:15:22 - mssql_python - DEBUG - Processed 1000 users +``` + +### Pattern 2: Plug Driver Logger Into Your Existing Logger + +If you already have application logging configured and want to integrate driver logs: + +```python +import logging +import mssql_python +from mssql_python.logging import logger as mssql_logger + +# Your existing application logger setup +app_logger = logging.getLogger('myapp') +app_logger.setLevel(logging.INFO) + +# Your existing handler and formatter +handler = logging.StreamHandler() +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +handler.setFormatter(formatter) +app_logger.addHandler(handler) + +# Now plug the driver logger into your handler +mssql_logger.addHandler(handler) # Use your handler +mssql_python.setup_logging() # Enable driver diagnostics + +# Use your app logger as normal +app_logger.info("Application starting") + +# Driver logs go to the same destination +conn = mssql_python.connect("Server=localhost;Database=test") + +app_logger.info("Querying database") +cursor = conn.cursor() +cursor.execute("SELECT * FROM users") + +app_logger.info("Application complete") +``` + +**Output shows both app and driver logs in your format:** +``` +2025-11-03 10:15:22 - myapp - INFO - Application starting +2025-11-03 10:15:22 - mssql_python - DEBUG - [Python] Initializing connection +2025-11-03 10:15:22 - mssql_python - DEBUG - [Python] Connection established +2025-11-03 10:15:22 - myapp - INFO - Querying database +2025-11-03 10:15:22 - mssql_python - DEBUG - [Python] Executing query +2025-11-03 10:15:22 - myapp - INFO - Application complete +``` + +**Key Benefits:** +- All logs go to your existing handlers (file, console, cloud, etc.) +- Use your existing formatters and filters +- Centralized log management +- No separate log files to manage + +### Pattern 3: Advanced - Custom Log Processing + +For advanced scenarios where you want to process driver logs programmatically: + +```python +import logging +import mssql_python +from mssql_python.logging import logger as mssql_logger + +class DatabaseAuditHandler(logging.Handler): + """Custom handler that audits database operations.""" + + def __init__(self): + super().__init__() + self.queries = [] + self.connections = [] + + def emit(self, record): + msg = record.getMessage() + + # Track queries + if 'Executing query' in msg: + self.queries.append({ + 'time': record.created, + 'query': msg + }) + + # Track connections + if 'Connection established' in msg: + self.connections.append({ + 'time': record.created, + 'level': record.levelname + }) + +# Setup audit handler +audit_handler = DatabaseAuditHandler() +mssql_logger.addHandler(audit_handler) +mssql_python.setup_logging() + +# Use the driver +conn = mssql_python.connect("Server=localhost;Database=test") +cursor = conn.cursor() +cursor.execute("SELECT * FROM users") +cursor.execute("SELECT * FROM orders") +conn.close() + +# Access audit data +print(f"Total queries executed: {len(audit_handler.queries)}") +print(f"Total connections: {len(audit_handler.connections)}") +for query in audit_handler.queries: + print(f" - {query['query']}") +``` + +## Common Patterns + +### Development Setup + +```python +import mssql_python + +# Both console and file - see everything +mssql_python.setup_logging(output='both') + +# Use the driver - see everything in console and file +conn = mssql_python.connect("Server=localhost;Database=test") +``` + +### Production Setup + +```python +import mssql_python + +# ⚠️ DO NOT enable logging in production without reason +# Logging adds overhead and should only be used for troubleshooting + +# If needed for specific troubleshooting: +# mssql_python.setup_logging() # Temporary only! +``` + +### CI/CD Pipeline Setup + +```python +import mssql_python + +# Stdout only (captured by CI system, no files) +mssql_python.setup_logging(output='stdout') + +# CI will capture all driver logs +conn = mssql_python.connect(connection_string) +``` + +### Debugging Specific Issues + +```python +import mssql_python + +# For ANY debugging - just enable logging (shows everything) +mssql_python.setup_logging(output='both') # See in console + save to file + +# Save debug logs to specific location for analysis +mssql_python.setup_logging(log_file_path="/tmp/mssql_debug.log") + +# For CI/CD troubleshooting +mssql_python.setup_logging(output='stdout') +``` + +### Integrate with Application Logging + +```python +import logging as py_logging +import mssql_python +from mssql_python.logging import logger as mssql_logger + +# Setup your application logger +app_logger = py_logging.getLogger('myapp') +app_logger.setLevel(py_logging.INFO) + +# Setup handler +handler = py_logging.StreamHandler() +handler.setFormatter(py_logging.Formatter('%(name)s - %(message)s')) +app_logger.addHandler(handler) + +# Plug driver logger into your handler +mssql_logger.addHandler(handler) +mssql_python.setup_logging() + +# Both logs go to same destination +app_logger.info("App started") +conn = mssql_python.connect("Server=localhost;Database=test") +app_logger.info("Database connected") +``` + +## Troubleshooting + +### No Log Output + +```python +import mssql_python +from mssql_python.logging import logger + +# Make sure you called setup_logging +mssql_python.setup_logging(output='stdout') # Force stdout output + +# Check logger level +print(f"Logger level: {logger.level}") +``` + +### Where is the Log File? + +```python +import mssql_python +from mssql_python.logging import logger + +# Enable logging first +mssql_python.setup_logging() + +# Then check location +print(f"Log file: {logger.log_file}") +# Output: ./mssql_python_logs/mssql_python_trace_20251103_101522_12345.log +``` + +### Logs Not Showing in CI/CD + +```python +# Use stdout for CI/CD systems +import mssql_python + +mssql_python.setup_logging(output='stdout') +# Now logs go to stdout and CI can capture them +``` + +## Best Practices + +1. **⚠️ Performance Warning**: Logging has overhead - only enable when troubleshooting + ```python + # ❌ DON'T enable logging by default + # ✅ DO enable only when investigating issues + ``` + +2. **Enable Early**: Configure logging before creating connections + ```python + mssql_python.setup_logging() # Do this first + conn = mssql_python.connect(...) # Then connect + ``` + +3. **Choose Right Output Destination**: + - **Development/Troubleshooting**: `output='both'` (see logs immediately + keep file) + - **CI/CD**: `output='stdout'` (no file clutter, captured by CI) + - **Customer debugging**: `output='file'` with custom path (default) + +4. **Log Files Auto-Rotate**: Files automatically rotate at 512MB, keeps 5 backups + +5. **Sanitization is Automatic**: Passwords are automatically redacted in logs + +6. **One-Line Setup**: Simple API: + ```python + mssql_python.setup_logging() # That's it! + ``` + +7. **Not for Performance Analysis**: Use profiler (future enhancement) for query performance, not logging + +## Examples + +### Complete Application Example + +```python +#!/usr/bin/env python3 +"""Example application with optional logging.""" + +import sys +import mssql_python +from mssql_python.logging import logger + +def main(debug: bool = False): + """Run the application with optional debug logging.""" + + # Setup logging only if debugging + if debug: + # Development: both file and console + mssql_python.setup_logging(output='both') + print(f"Logging to: {logger.log_file}") + + # Connect to database + conn = mssql_python.connect( + server='localhost', + database='testdb', + trusted_connection='yes' + ) + + # Execute query + cursor = conn.cursor() + cursor.execute("SELECT TOP 10 * FROM users WHERE active = ?", (1,)) + + # Process results + for row in cursor: + print(f"User: {row.username}") + + # Cleanup + cursor.close() + conn.close() + +if __name__ == '__main__': + import sys + debug = '--debug' in sys.argv + main(debug=debug) +``` + +## Performance Considerations + +- **⚠️ Logging Has Overhead**: When enabled, logging adds ~2-5% performance overhead + ```python + # Logging disabled by default - no overhead + conn = mssql_python.connect(...) # Full performance + + # Enable only when troubleshooting + mssql_python.setup_logging() # Now has ~2-5% overhead + ``` + +- **Not for Performance Analysis**: Do NOT use logging to measure query performance + - Logging itself adds latency + - Use profiler (future enhancement) for accurate performance metrics + +- **Lazy Initialization**: Handlers are only created when `setup_logging()` is called + +- **File I/O**: File logging has minimal overhead with buffering + +- **Automatic Rotation**: Files rotate at 512MB to prevent disk space issues + +## Design Philosophy + +**Simple and Purposeful** + +1. **All or Nothing**: No levels to choose from - either debug everything or don't log +2. **Troubleshooting Tool**: Logging is for diagnosing problems, not production monitoring +3. **Performance Conscious**: Clear warning that logging has overhead +4. **Future-Proof**: Profiler (future) will handle performance analysis properly + +### Minimal API Surface + +Most users only need one line: + +```python +mssql_python.setup_logging() # That's it! +``` + +This follows the [Zen of Python](https://www.python.org/dev/peps/pep-0020/): "Simple is better than complex." + +## Support + +For issues or questions: +- GitHub Issues: [microsoft/mssql-python](https://github.com/microsoft/mssql-python) +- Documentation: See `MSSQL-Python-Logging-Design.md` for technical details diff --git a/LOGGING_TROUBLESHOOTING_GUIDE.md b/LOGGING_TROUBLESHOOTING_GUIDE.md new file mode 100644 index 00000000..9d04f1a6 --- /dev/null +++ b/LOGGING_TROUBLESHOOTING_GUIDE.md @@ -0,0 +1,1432 @@ +# mssql-python Logging Troubleshooting Guide + +**Version:** 1.0 +**Last Updated:** November 4, 2025 + +--- + +## Table of Contents + +1. [Quick Reference](#quick-reference) +2. [Enable Debug Logging](#enable-debug-logging) +3. [Common Customer Issues](#common-customer-issues) +4. [Step-by-Step Troubleshooting Workflows](#step-by-step-troubleshooting-workflows) +5. [Permission Issues](#permission-issues) +6. [Log Collection Guide](#log-collection-guide) +7. [Log Analysis](#log-analysis) +8. [Escalation Criteria](#escalation-criteria) +9. [FAQ](#faq) +10. [Scripts & Commands](#scripts--commands) + +--- + +## Quick Reference + +### Fastest Way to Enable Logging + +```python +import mssql_python + +# Enable logging - shows everything +mssql_python.setup_logging(output='both') +``` + +This enables logging with: +- ✅ File output (in `./mssql_python_logs/` folder) +- ✅ Console output (immediate visibility) +- ✅ Debug level (everything) + +### Logging Philosophy + +mssql-python uses an **all-or-nothing** approach: +- **One Level**: DEBUG level only - no level categorization +- **All or Nothing**: When enabled, you see EVERYTHING +- **Troubleshooting Focus**: Turn on when something breaks, off otherwise + +### Output Modes + +| Mode | Value | Behavior | Use Case | +|------|-------|----------|----------| +| **File** | `'file'` | Logs to file only | Default, production | +| **Stdout** | `'stdout'` | Logs to console only | No file access | +| **Both** | `'both'` | Logs to file + console | Active troubleshooting | + +--- + +## Enable Debug Logging + +The mssql-python driver includes a comprehensive logging system that captures detailed information about driver operations, SQL queries, parameters, and internal state. + +### Quick Start + +Enable logging with one line before creating connections: + +```python +import mssql_python + +# Enable logging - shows EVERYTHING +mssql_python.setup_logging() + +# Use the driver - all operations are now logged +conn = mssql_python.connect("Server=localhost;Database=test") +# Log file: ./mssql_python_logs/mssql_python_trace_*.log +``` + +### Output Options + +Control where logs are written: + +```python +# File only (default) - logs saved to file +mssql_python.setup_logging() + +# Console only - logs printed to stdout +mssql_python.setup_logging(output='stdout') + +# Both file and console +mssql_python.setup_logging(output='both') + +# Custom file path (must use .txt, .log, or .csv extension) +mssql_python.setup_logging(log_file_path="/var/log/myapp/debug.log") +``` + +### What Gets Logged + +When enabled, logging shows **everything** at DEBUG level: + +- ✅ **Connection operations**: Opening, closing, configuration +- ✅ **SQL queries**: Full query text and parameters +- ✅ **Internal operations**: ODBC calls, handle management, memory allocations +- ✅ **Error details**: Exceptions with stack traces and error codes +- ✅ **Thread tracking**: OS native thread IDs for multi-threaded debugging + +### Log Format + +Logs use comma-separated format with structured fields: + +``` +# MSSQL-Python Driver Log | Script: main.py | PID: 12345 | Log Level: DEBUG | Python: 3.13.7 | Start: 2025-11-06 10:30:15 +Timestamp, ThreadID, Level, Location, Source, Message +2025-11-06 10:30:15.100, 8581947520, DEBUG, connection.py:156, Python, Connection opened +2025-11-06 10:30:15.101, 8581947520, DEBUG, connection.cpp:22, DDBC, Allocating ODBC environment handle +2025-11-06 10:30:15.102, 8581947520, DEBUG, cursor.py:89, Python, Executing query: SELECT * FROM users WHERE id = ? +2025-11-06 10:30:15.103, 8581947520, DEBUG, cursor.py:90, Python, Query parameters: [42] +``` + +**Field Descriptions:** +- **Timestamp**: Precise time with milliseconds +- **ThreadID**: OS native thread ID (matches debugger thread IDs) +- **Level**: Always DEBUG when logging enabled +- **Location**: Source file and line number +- **Source**: Python (Python layer) or DDBC (C++ layer) +- **Message**: Operation details, queries, parameters, etc. + +**Why Thread IDs?** +- Track operations in multi-threaded applications +- Distinguish concurrent connections/queries +- Correlate with debugger thread views +- Filter logs by specific thread + +### Performance Notes + +⚠️ **Important**: Logging adds ~2-5% overhead. Enable only when troubleshooting. + +```python +# ❌ DON'T enable by default in production +# ✅ DO enable only when diagnosing issues +``` + +### Using Driver Logger in Your Application + +Integrate the driver's logger into your own code: + +```python +import mssql_python +from mssql_python.logging import driver_logger + +# Enable logging +mssql_python.setup_logging() + +# Use driver_logger in your application +driver_logger.debug("[App] Starting data processing") +driver_logger.info("[App] Processing complete") +driver_logger.warning("[App] Resource usage high") +driver_logger.error("[App] Failed to process record") + +# Your logs appear in the same file as driver logs +``` + +### Common Troubleshooting + +**No log output?** +```python +# Force stdout to verify logging works +mssql_python.setup_logging(output='stdout') +``` + +**Where is the log file?** +```python +from mssql_python import driver_logger +mssql_python.setup_logging() +# Access log file path from driver_logger handlers if needed +``` + +**Logs not showing in CI/CD?** +```python +# Use stdout for CI/CD pipelines +mssql_python.setup_logging(output='stdout') +``` + +**Invalid file extension error?** +```python +# Only .txt, .log, or .csv extensions allowed +mssql_python.setup_logging(log_file_path="/tmp/debug.log") # ✓ +mssql_python.setup_logging(log_file_path="/tmp/debug.json") # ✗ ValueError +``` + +--- + +## Common Customer Issues + +### Issue 1: "I can't connect to the database" + +**Symptoms:** +- Connection timeout +- Authentication failures +- Network errors + +**Solution Steps:** + +1. **Enable logging to see connection attempts:** + +```python +import mssql_python + +# Enable logging +mssql_python.setup_logging(output='both') + +# Then run customer's connection code +conn = mssql_python.connect(connection_string) +``` + +2. **What to look for in logs:** +- `[Python] Connecting to server: ` - Connection initiated +- `[Python] Connection established` - Success +- Error messages with connection details + +3. **Common log patterns:** + +**Success:** +``` +2025-11-04 10:30:15 [CONN-12345-67890-1] - DEBUG - connection.py:42 - [Python] Connecting to server: localhost +2025-11-04 10:30:15 [CONN-12345-67890-1] - DEBUG - connection.py:89 - [Python] Connection established +``` + +**Failure (wrong server):** +``` +2025-11-04 10:30:15 [CONN-12345-67890-1] - DEBUG - connection.py:42 - [Python] Connecting to server: wrongserver +2025-11-04 10:30:20 [CONN-12345-67890-1] - ERROR - connection.py:156 - [Python] Connection failed: timeout +``` + +**Action:** Check server name, network connectivity, firewall rules + +--- + +### Issue 2: "Query returns wrong results" + +**Symptoms:** +- Incorrect data returned +- Missing rows +- Wrong column values + +**Solution Steps:** + +1. **Enable logging to see SQL + parameters:** + +```python +import mssql_python + +# Enable logging +mssql_python.setup_logging(output='both') + +# Run customer's query +cursor.execute("SELECT * FROM users WHERE id = ?", (user_id,)) +``` + +2. **What to look for:** +- Actual SQL being executed +- Parameter values being passed +- Parameter types + +3. **Common issues:** +- Wrong parameter value: `Parameter 1: value=999` (expected 123) +- Wrong parameter order: `Parameter 1: value='John', Parameter 2: value=123` (swapped) +- Type mismatch: `Parameter 1: type=str, value='123'` (should be int) + +**Action:** Verify SQL statement and parameter values match customer expectations + +--- + +### Issue 3: "Query is very slow" + +**Symptoms:** +- Long execution time +- Timeouts +- Performance degradation + +**Solution Steps:** + +1. **Enable logging with timing:** + +```python +import mssql_python +import time + +mssql_python.setup_logging(output='both') + +start = time.time() +cursor.execute("SELECT * FROM large_table WHERE ...") +rows = cursor.fetchall() +end = time.time() + +print(f"Query took {end - start:.2f} seconds") +``` + +2. **What to look for in logs:** +- Query execution timestamp +- Large result sets: `Fetched 1000000 rows` +- Multiple round trips to database + +3. **Common patterns:** + +**Inefficient query:** +``` +2025-11-04 10:30:15 - DEBUG - cursor.py:28 - [Python] Executing query: SELECT * FROM huge_table +2025-11-04 10:35:20 - DEBUG - cursor.py:89 - [Python] Query completed, 5000000 rows fetched +``` + +**Action:** Check if query can be optimized, add WHERE clause, use pagination + +--- + +### Issue 4: "I get a parameter binding error" + +**Symptoms:** +- `Invalid parameter type` +- `Cannot convert parameter` +- Data truncation errors + +**Solution Steps:** + +1. **Enable logging to see parameter binding:** + +```python +import mssql_python + +mssql_python.setup_logging(output='both') + +cursor.execute("SELECT * FROM table WHERE col = ?", (param,)) +``` + +2. **What to look for:** +- `_map_sql_type: Mapping param index=0, type=` +- `_map_sql_type: INT detected` (or other type) +- `_map_sql_type: INT -> BIGINT` (type conversion) + +3. **Example log output:** + +``` +2025-11-04 10:30:15 - DEBUG - cursor.py:310 - _map_sql_type: Mapping param index=0, type=Decimal +2025-11-04 10:30:15 - DEBUG - cursor.py:385 - _map_sql_type: DECIMAL detected - index=0 +2025-11-04 10:30:15 - DEBUG - cursor.py:406 - _map_sql_type: DECIMAL precision calculated - index=0, precision=18 +``` + +**Action:** Verify parameter type matches database column type, convert if needed + +--- + +### Issue 5: "executemany fails with batch data" + +**Symptoms:** +- Batch insert/update fails +- Some rows succeed, others fail +- Transaction rollback + +**Solution Steps:** + +1. **Enable logging to see batch operations:** + +```python +import mssql_python +mssql_python.setup_logging(output='both') + +data = [(1, 'Alice'), (2, 'Bob'), (3, 'Charlie')] +cursor.executemany("INSERT INTO users (id, name) VALUES (?, ?)", data) +``` + +2. **What to look for:** +- `executemany: Starting - batch_count=` +- Individual parameter sets being processed +- Errors on specific batch items + +**Action:** Check if all rows in batch have consistent types and valid data + +--- + +## Step-by-Step Troubleshooting Workflows + +### Workflow 1: Connection Issues + +**Customer says:** "I can't connect to my database" + +**Step 1: Enable logging** +```python +import mssql_python +mssql_python.setup_logging(output='both') +``` + +**Step 2: Attempt connection** +```python +import mssql_python +try: + conn = mssql_python.connect( + server='servername', + database='dbname', + username='user', + password='pass' + ) + print("✅ Connected successfully!") +except Exception as e: + print(f"❌ Connection failed: {e}") +``` + +**Step 3: Check console output** +Look for: +- Server name in logs matches expected server +- No "connection timeout" errors +- No "login failed" errors + +**Step 4: Check log file** +```python +print(f"Log file: {logging.logger.log_file}") +``` +Open the file and search for "ERROR" or "Connection" + +**Step 5: Collect information** +- Server name (sanitized) +- Database name +- Authentication method (Windows/SQL) +- Error message +- Log file + +**Escalate if:** +- Logs show "connection established" but customer says it fails +- Unusual error messages +- Consistent timeout at specific interval + +--- + +### Workflow 2: Query Problems + +**Customer says:** "My query doesn't work" + +**Step 1: Enable logging** +```python +import mssql_python +mssql_python.setup_logging(output='both') +``` + +**Step 2: Run the query** +```python +cursor = conn.cursor() +try: + cursor.execute("SELECT * FROM table WHERE id = ?", (123,)) + rows = cursor.fetchall() + print(f"✅ Fetched {len(rows)} rows") +except Exception as e: + print(f"❌ Query failed: {e}") +``` + +**Step 3: Check logs for:** +- Exact SQL statement executed +- Parameter values (are they what customer expects?) +- Row count returned + +**Step 4: Verify customer expectations** +Ask: +- "Is the SQL statement correct?" +- "Are the parameter values correct?" +- "How many rows should be returned?" + +**Step 5: Collect information** +- SQL statement (sanitized) +- Parameter values (sanitized) +- Expected vs actual results +- Error message (if any) +- Log file + +**Escalate if:** +- SQL and parameters look correct but results are wrong +- Driver returns different results than SSMS +- Reproducible data corruption + +--- + +### Workflow 3: Performance Issues + +**Customer says:** "Queries are too slow" + +**Step 1: Enable timing measurements** +```python +import mssql_python +import time + +mssql_python.setup_logging(output='both') + +start = time.time() +cursor.execute("SELECT * FROM large_table") +rows = cursor.fetchall() +elapsed = time.time() - start + +print(f"Query took {elapsed:.2f} seconds, fetched {len(rows)} rows") +``` + +**Step 2: Check log file for patterns** +```python +print(f"Log file: {logging.logger.log_file}") +``` + +Look for: +- Very large row counts: `Fetched 1000000 rows` +- Multiple queries: Customer might be in a loop +- Long timestamps between execute and fetch + +**Step 3: Compare logging overhead** + +Run with logging disabled: +```python +# Don't call setup_logging() - logging disabled by default +start = time.time() +cursor.execute("SELECT * FROM large_table") +rows = cursor.fetchall() +elapsed = time.time() - start +print(f"Without logging: {elapsed:.2f} seconds") +``` + +If significantly faster, logging overhead is the issue. + +**Step 4: Profile the query** +Ask customer to run same query in SSMS or Azure Data Studio: +- If fast there: Driver issue (escalate) +- If slow there: Query optimization needed (not driver issue) + +**Step 5: Collect information** +- Query execution time +- Row count +- Query complexity +- Database server specs +- Network latency +- Logging level used + +**Escalate if:** +- Query is fast in SSMS but slow with driver +- Same query was fast before, slow now +- Logging overhead exceeds 10% with logging enabled + +--- + +## Permission Issues + +### Issue: Customer Can't Create Log Files + +**Symptom:** Error when enabling logging +``` +PermissionError: [Errno 13] Permission denied: './mssql_python_logs/mssql_python_trace_...' +``` + +**Root Cause:** No write permission in current directory or specified path + +**Solutions:** + +#### Solution 1: Use STDOUT Only (No File Access Needed) + +```python +import mssql_python + +# Console output only - no file created +mssql_python.setup_logging(output='stdout') + +# Customer can copy console output to share with you +``` + +**Advantages:** +- ✅ No file permissions required +- ✅ Immediate visibility +- ✅ Works in restricted environments (Docker, CI/CD) + +**Disadvantages:** +- ❌ Output lost when console closed +- ❌ Large logs hard to manage in console + +--- + +#### Solution 2: Use Temp Directory + +```python +import tempfile +import os +import mssql_python + +# Get temp directory (usually writable by all users) +temp_dir = tempfile.gettempdir() +log_file = os.path.join(temp_dir, "mssql_python_debug.log") + +mssql_python.setup_logging(log_file_path=log_file) +print(f"Logging to: {log_file}") + +# On Windows: Usually C:\Users\\AppData\Local\Temp\mssql_python_debug.log +# On Linux/Mac: Usually /tmp/mssql_python_debug.log +``` + +**Advantages:** +- ✅ Temp directories are usually writable +- ✅ Log file persists during session +- ✅ Easy to locate and share + +--- + +#### Solution 3: Use User Home Directory + +```python +import os +from pathlib import Path +import mssql_python + +# User home directory - always writable by user +home_dir = Path.home() +log_dir = home_dir / "mssql_python_logs" +log_dir.mkdir(exist_ok=True) + +log_file = log_dir / "debug.log" +mssql_python.setup_logging(log_file_path=str(log_file)) +print(f"Logging to: {log_file}") + +# On Windows: C:\Users\\mssql_python_logs\debug.log +# On Linux/Mac: /home//mssql_python_logs/debug.log +``` + +**Advantages:** +- ✅ Always writable (it's user's home) +- ✅ Logs persist across sessions +- ✅ Easy for user to find + +--- + +#### Solution 4: Custom Writable Path + +Ask customer where they have write access: + +```python +import mssql_python + +# Ask customer: "Where can you create files?" +# Example paths: +# - Desktop: "C:/Users/username/Desktop/mssql_logs" +# - Documents: "C:/Users/username/Documents/mssql_logs" +# - Network share: "//server/share/logs" + +custom_path = "C:/Users/john/Desktop/mssql_debug.log" +mssql_python.setup_logging(log_file_path=custom_path) +print(f"Logging to: {custom_path}") +``` + +--- + +#### Solution 5: Use BOTH Mode with Temp File + +Best of both worlds: + +```python +import tempfile +import os +import mssql_python + +temp_dir = tempfile.gettempdir() +log_file = os.path.join(temp_dir, "mssql_python_debug.log") + +# Both console (immediate) and file (persistent) +mssql_python.setup_logging(output='both', log_file_path=log_file) + +print(f"✅ Logging to console AND file: {log_file}") +print("You can see logs immediately, and share the file later!") +``` + +--- + +### Testing Write Permissions + +Help customer test if they can write to a location: + +```python +import os +from pathlib import Path + +def test_write_permission(path): + """Test if customer can write to a directory.""" + try: + test_file = Path(path) / "test_write.txt" + test_file.write_text("test") + test_file.unlink() # Delete test file + return True, "✅ Write permission OK" + except Exception as e: + return False, f"❌ Cannot write: {e}" + +# Test current directory +can_write, msg = test_write_permission(".") +print(f"Current directory: {msg}") + +# Test temp directory +import tempfile +temp_dir = tempfile.gettempdir() +can_write, msg = test_write_permission(temp_dir) +print(f"Temp directory ({temp_dir}): {msg}") + +# Test home directory +home_dir = Path.home() +can_write, msg = test_write_permission(home_dir) +print(f"Home directory ({home_dir}): {msg}") +``` + +--- + +### Issue: Log Files Too Large + +**Symptom:** Log files consuming too much disk space + +**Solution 1: Logging is All-or-Nothing** + +```python +# Logging shows everything when enabled +mssql_python.setup_logging() # All operations logged at DEBUG level +``` + +Logging in mssql-python uses a simple DEBUG level - no granular levels to choose from. + +**Solution 2: Check Rotation Settings** + +Log files automatically rotate at 512MB with 5 backups. This means max ~2.5GB total. + +If customer needs smaller files: +```python +import logging as py_logging +from mssql_python import driver_logger + +# After enabling logging, modify the handler +mssql_python.setup_logging() + +for handler in driver_logger.handlers: + if isinstance(handler, py_logging.handlers.RotatingFileHandler): + handler.maxBytes = 50 * 1024 * 1024 # 50MB instead of 512MB + handler.backupCount = 2 # 2 backups instead of 5 +``` + +**Solution 3: Don't Enable Logging Unless Troubleshooting** + +```python +# ❌ DON'T enable by default +# mssql_python.setup_logging() # Comment out when not needed + +# ✅ DO enable only when troubleshooting +if debugging: + mssql_python.setup_logging() +``` + +--- + +## Log Collection Guide + +### How to Collect Logs from Customer + +**Step 1: Ask customer to enable logging** + +Send them this code: +```python +import mssql_python +import tempfile +import os + +# Use temp directory (always writable) +log_file = os.path.join(tempfile.gettempdir(), "mssql_python_debug.log") +mssql_python.setup_logging(output='both', log_file_path=log_file) + +print(f"✅ Logging enabled") +print(f"📂 Log file: {log_file}") +print("Please run your code that reproduces the issue, then send me the log file.") +``` + +**Step 2: Customer reproduces issue** + +Customer runs their code that has the problem. + +**Step 3: Customer finds log file** + +The code above prints the log file path. Customer can: +- Copy the path +- Open in Notepad/TextEdit +- Attach to support ticket + +**Step 4: Customer sends log file** + +Options: +- Email attachment +- Support portal upload +- Paste in ticket (if small) + +--- + +### What to Ask For + +**Minimum information:** +1. ✅ Log file (with logging enabled) +2. ✅ Code snippet that reproduces issue (sanitized) +3. ✅ Error message (if any) +4. ✅ Expected vs actual behavior + +**Nice to have:** +5. Python version: `python --version` +6. Driver version: `pip show mssql-python` +7. Operating system: Windows/Linux/Mac +8. Database server version: SQL Server 2019/2022, Azure SQL, etc. + +--- + +### Sample Email Template for Customer + +``` +Subject: mssql-python Logging Instructions + +Hi [Customer], + +To help troubleshoot your issue, please enable logging and send us the log file. + +1. Add these lines at the start of your code: + +import mssql_python +import tempfile +import os + +log_file = os.path.join(tempfile.gettempdir(), "mssql_python_debug.log") +mssql_python.setup_logging(output='both', log_file_path=log_file) +print(f"Log file: {log_file}") + +2. Run your code that reproduces the issue + +3. Find the log file (path printed in step 1) + +4. Send us: + - The log file + - Your code (remove any passwords!) + - The error message you see + +This will help us diagnose the problem quickly. + +Thanks! +``` + +--- + +## Log Analysis + +### Reading Log Files + +**Log Format:** +``` +2025-11-04 10:30:15,123 [CONN-12345-67890-1] - DEBUG - connection.py:42 - [Python] Message +│ │ │ │ │ +│ │ │ │ └─ Log message +│ │ │ └─ Source file:line +│ │ └─ Log level (always DEBUG) +│ └─ Trace ID (PREFIX-PID-ThreadID-Counter) +└─ Timestamp (YYYY-MM-DD HH:MM:SS,milliseconds) +``` + +**Trace ID Components:** +- `CONN-12345-67890-1` = Connection, Process 12345, Thread 67890, Sequence 1 +- `CURS-12345-67890-2` = Cursor, Process 12345, Thread 67890, Sequence 2 + +**Why Trace IDs matter:** +- Multi-threaded apps: Distinguish logs from different threads +- Multiple connections: Track which connection did what +- Debugging: Filter logs with `grep "CONN-12345-67890-1" logfile.log` + +--- + +### Common Log Patterns + +#### Pattern 1: Successful Connection + +``` +2025-11-04 10:30:15,100 [CONN-12345-67890-1] - DEBUG - connection.py:42 - [Python] Connecting to server: localhost +2025-11-04 10:30:15,250 [CONN-12345-67890-1] - DEBUG - connection.py:89 - [Python] Connection established +``` + +**Interpretation:** Connection succeeded in ~150ms + +--- + +#### Pattern 2: Query Execution + +``` +2025-11-04 10:30:16,100 [CURS-12345-67890-2] - DEBUG - cursor.py:1040 - execute: Starting - operation_length=45, param_count=2, use_prepare=False +2025-11-04 10:30:16,350 [CURS-12345-67890-2] - DEBUG - cursor.py:1200 - [Python] Query completed, 42 rows fetched +``` + +**Interpretation:** +- Query took ~250ms +- Had 2 parameters +- Returned 42 rows + +--- + +#### Pattern 3: Parameter Binding + +``` +2025-11-04 10:30:16,100 [CURS-12345-67890-2] - DEBUG - cursor.py:1063 - execute: Setting query timeout=30 seconds +2025-11-04 10:30:16,105 [CURS-12345-67890-2] - DEBUG - cursor.py:310 - _map_sql_type: Mapping param index=0, type=int +2025-11-04 10:30:16,106 [CURS-12345-67890-2] - DEBUG - cursor.py:335 - _map_sql_type: INT detected - index=0, min=100, max=100 +2025-11-04 10:30:16,107 [CURS-12345-67890-2] - DEBUG - cursor.py:339 - _map_sql_type: INT -> TINYINT - index=0 +``` + +**Interpretation:** +- Parameter 0 is an integer with value 100 +- Driver chose TINYINT (smallest int type that fits) + +--- + +#### Pattern 4: Error + +``` +2025-11-04 10:30:16,100 [CURS-12345-67890-2] - DEBUG - cursor.py:1040 - execute: Starting - operation_length=45, param_count=2, use_prepare=False +2025-11-04 10:30:16,200 [CURS-12345-67890-2] - ERROR - cursor.py:1500 - [Python] Query failed: Invalid object name 'users' +``` + +**Interpretation:** +- Query tried to access table 'users' that doesn't exist +- Failed after 100ms + +--- + +### Searching Logs Effectively + +**Find all errors:** +```bash +grep "ERROR" mssql_python_trace_*.log +``` + +**Find specific connection:** +```bash +grep "CONN-12345-67890-1" mssql_python_trace_*.log +``` + +**Find slow queries (multi-second timestamps):** +```bash +grep "Query completed" mssql_python_trace_*.log +``` + +**Find parameter issues:** +```bash +grep "_map_sql_type" mssql_python_trace_*.log | grep "DEBUG\|ERROR" +``` + +**On Windows PowerShell:** +```powershell +Select-String -Path "mssql_python_trace_*.log" -Pattern "ERROR" +``` + +--- + +### Red Flags in Logs + +🚩 **Multiple connection attempts:** +``` +10:30:15 - Connecting to server: localhost +10:30:20 - Connection failed: timeout +10:30:21 - Connecting to server: localhost +10:30:26 - Connection failed: timeout +``` +→ Network or firewall issue + +🚩 **Massive row counts:** +``` +10:30:15 - Query completed, 5000000 rows fetched +``` +→ Query needs pagination or WHERE clause + +🚩 **Repeated failed queries:** +``` +10:30:15 - ERROR - Query failed: Invalid column name 'xyz' +10:30:16 - ERROR - Query failed: Invalid column name 'xyz' +10:30:17 - ERROR - Query failed: Invalid column name 'xyz' +``` +→ Customer code in a retry loop with broken query + +🚩 **Type conversion warnings:** +``` +10:30:15 - DEBUG - _map_sql_type: DECIMAL precision too high - index=0, precision=50 +``` +→ Customer passing Decimal with precision exceeding SQL Server limits (38) + +🚩 **Password in logs (should never happen):** +``` +10:30:15 - Connection string: Server=...;PWD=***REDACTED*** +``` +✅ Good - password sanitized + +``` +10:30:15 - Connection string: Server=...;PWD=MyPassword123 +``` +❌ BAD - sanitization failed, escalate immediately + +--- + +## Escalation Criteria + +### Escalate to Engineering If: + +1. **Data Corruption** + - Logs show correct data, customer sees wrong data + - Reproducible with minimal code + - Not an application logic issue + +2. **Driver Crashes** + - Python crashes/segfaults + - C++ exceptions in logs + - Memory access violations + +3. **Performance Regression** + - Query is fast in SSMS, slow in driver + - Same query was fast before, slow now + - Logging overhead exceeds 10% with logging enabled + +4. **Security Issues** + - Passwords not sanitized in logs + - SQL injection vulnerability + - Authentication bypass + +5. **Inconsistent Behavior** + - Works on one machine, fails on another (same environment) + - Intermittent failures with no pattern + - Different results between driver and SSMS + +6. **Cannot Reproduce** + - Customer provides logs showing issue + - You cannot reproduce with same code + - Issue appears to be environment-specific but customer insists environment is standard + +### Escalation Package + +When escalating, include: + +1. ✅ **Log files** (logging enabled) +2. ✅ **Minimal reproduction code** (sanitized) +3. ✅ **Customer environment:** + - Python version + - Driver version (`pip show mssql-python`) + - OS (Windows/Linux/Mac) + version + - Database server (SQL Server version, Azure SQL, etc.) +4. ✅ **Steps to reproduce** +5. ✅ **Expected vs actual behavior** +6. ✅ **Your analysis** (what you've tried, why you're escalating) +7. ✅ **Customer impact** (severity, business impact) + +### Do NOT Escalate If: + +1. ❌ Customer's SQL query is incorrect (not a driver issue) +2. ❌ Database permissions issue (customer can't access table) +3. ❌ Network connectivity issue (firewall, DNS, etc.) +4. ❌ Application logic bug (customer's code issue) +5. ❌ Customer hasn't provided logs yet +6. ❌ You haven't tried basic troubleshooting steps + +--- + +## FAQ + +### Q1: Why do I see `[Python]` in log messages? + +**A:** This prefix distinguishes Python-side operations from C++ internal operations. You may also see `[DDBC]` for C++ driver operations. + +``` +[Python] Connecting to server - Python layer +[DDBC] Allocating connection handle - C++ layer +``` + +--- + +### Q2: Customer says logging "doesn't work" + +**Checklist:** + +1. Did they call `setup_logging()`? + ```python + # ❌ Won't work - logging not enabled + import mssql_python + conn = mssql_python.connect(...) + + # ✅ Will work - logging enabled + import mssql_python + mssql_python.setup_logging() + conn = mssql_python.connect(...) + ``` + +2. Are they looking in the right place? + - Default: `./mssql_python_logs/` directory + - Custom path if specified with `log_file_path` + +3. Do they have write permissions? +3. Do they have write permissions? + ```python + # Try STDOUT instead + mssql_python.setup_logging(output='stdout') + ``` + +--- + +### Q3: Log file is empty + +**Possible causes:** + +1. **Logging enabled after operations:** Must enable BEFORE operations + ```python + # ❌ Wrong order + conn = mssql_python.connect(...) # Not logged + mssql_python.setup_logging() # Too late! + + # ✅ Correct order + mssql_python.setup_logging() # Enable first + conn = mssql_python.connect(...) # Now logged + ``` + +2. **Python buffering:** Logs may not flush until script ends + ```python + # Force flush after operations + from mssql_python import driver_logger + for handler in driver_logger.handlers: + handler.flush() + ``` + +3. **Wrong log file:** Customer looking at old file + +--- + +### Q4: How much overhead does logging add? + +**Performance impact:** + +| Level | Overhead | File Size (1000 queries) | +|-------|----------|-------------------------| +| DISABLED | 0% | 0 KB | +| DEBUG (enabled) | 2-10% | ~100-500 KB | + +**Note:** Logging is all-or-nothing in mssql-python - when enabled, all operations are logged at DEBUG level. + +--- + +### Q5: Can customer use their own log file name? + +**A:** Yes! They can specify any path: + +```python +# Custom name in default folder +mssql_python.setup_logging(log_file_path="./mssql_python_logs/my_app.log") + +# Completely custom path +mssql_python.setup_logging(log_file_path="C:/Logs/database_debug.log") + +# Only .txt, .log, .csv extensions allowed +mssql_python.setup_logging(log_file_path="./mssql_python_logs/debug.csv") +``` + +--- + +### Q6: Are passwords visible in logs? + +**A:** No! Passwords are automatically sanitized: + +``` +# In logs you'll see: +Connection string: Server=localhost;Database=test;UID=admin;PWD=***REDACTED*** +``` + +**If you see actual passwords in logs, ESCALATE IMMEDIATELY** - this is a security bug. + +--- + +### Q7: Can we send logs to our logging system? + +**A:** Yes! The driver uses standard Python logging, so you can add custom handlers: + +```python +import mssql_python +from mssql_python import driver_logger + +# Add Splunk/DataDog/CloudWatch handler +custom_handler = MySplunkHandler(...) +driver_logger.addHandler(custom_handler) + +# Now logs go to both file and your system +mssql_python.setup_logging() +``` + +--- + +### Q8: How long are logs kept? + +**A:** +- Files rotate at 512MB +- Keeps 5 backup files +- Total max: ~2.5GB +- No automatic deletion - customer must clean up old files + +--- + +### Q9: Customer has multiple Python scripts - which one generates which logs? + +**A:** Each script creates its own log file with timestamp + PID: + +``` +mssql_python_logs/ +├── mssql_python_trace_20251104_100000_12345.log ← Script 1 (PID 12345) +├── mssql_python_trace_20251104_100100_12346.log ← Script 2 (PID 12346) +└── mssql_python_trace_20251104_100200_12347.log ← Script 3 (PID 12347) +``` + +Trace IDs also include PID for correlation. + +--- + +### Q10: What if customer is using Docker/Kubernetes? + +**Solution:** Use STDOUT mode so logs go to container logs: + +```python +import mssql_python +mssql_python.setup_logging(output='stdout') + +# Logs appear in: docker logs +# or: kubectl logs +``` + +--- + +## Scripts & Commands + +### Script 1: Quick Diagnostic + +Send this to customer for quick info collection: + +```python +""" +Quick Diagnostic Script for mssql-python +Collects environment info and tests logging +""" + +import sys +import platform +import tempfile +import os + +print("=" * 70) +print("mssql-python Diagnostic Script") +print("=" * 70) +print() + +# Environment info +print("📋 Environment Information:") +print(f" Python version: {sys.version}") +print(f" Platform: {platform.system()} {platform.release()}") +print(f" Architecture: {platform.machine()}") +print() + +# Driver version +try: + import mssql_python + print(f" mssql-python version: {mssql_python.__version__}") +except Exception as e: + print(f" ❌ Cannot import mssql-python: {e}") + sys.exit(1) +print() + +# Test logging +print("🔧 Testing Logging:") + +temp_dir = tempfile.gettempdir() +log_file = os.path.join(temp_dir, "mssql_python_diagnostic.log") + +try: + mssql_python.setup_logging(output='both', log_file_path=log_file) + print(f" ✅ Logging enabled successfully") + print(f" 📂 Log file: {log_file}") +except Exception as e: + print(f" ❌ Logging failed: {e}") + print(f" Try STDOUT mode instead:") + print(f" mssql_python.setup_logging(output='stdout')") +print() + +# Test connection (if connection string provided) +conn_str = os.getenv("DB_CONNECTION_STRING") +if conn_str: + print("🔌 Testing Connection:") + try: + conn = mssql_python.connect(conn_str) + print(" ✅ Connection successful") + cursor = conn.cursor() + cursor.execute("SELECT @@VERSION") + version = cursor.fetchone()[0] + print(f" Database: {version[:80]}...") + cursor.close() + conn.close() + except Exception as e: + print(f" ❌ Connection failed: {e}") + print() +else: + print("ℹ️ Set DB_CONNECTION_STRING env var to test connection") + print() + +print("=" * 70) +print("✅ Diagnostic complete!") +print(f"📂 Log file: {log_file}") +print("Please send this output and the log file to support.") +print("=" * 70) +``` + +--- + +### Script 2: Permission Tester + +Test where customer can write log files: + +```python +""" +Test write permissions in various directories +""" + +import os +import tempfile +from pathlib import Path + +def test_write(path, name): + """Test if we can write to a path.""" + try: + test_file = Path(path) / "test_write.txt" + test_file.write_text("test") + test_file.unlink() + print(f" ✅ {name}: {path}") + return True + except Exception as e: + print(f" ❌ {name}: {path}") + print(f" Error: {e}") + return False + +print("Testing write permissions...") +print() + +# Current directory +test_write(".", "Current directory") + +# Temp directory +test_write(tempfile.gettempdir(), "Temp directory") + +# Home directory +test_write(Path.home(), "Home directory") + +# Desktop (if exists) +desktop = Path.home() / "Desktop" +if desktop.exists(): + test_write(desktop, "Desktop") + +# Documents (if exists) +documents = Path.home() / "Documents" +if documents.exists(): + test_write(documents, "Documents") + +print() +print("Use one of the ✅ paths for log files!") +``` + +--- + +### Script 3: Log Analyzer + +Help analyze log files: + +```python +""" +Simple log analyzer for mssql-python logs +""" + +import sys +from pathlib import Path + +if len(sys.argv) < 2: + print("Usage: python analyze_log.py ") + sys.exit(1) + +log_file = Path(sys.argv[1]) +if not log_file.exists(): + print(f"❌ File not found: {log_file}") + sys.exit(1) + +print(f"📊 Analyzing: {log_file}") +print("=" * 70) +print() + +with open(log_file) as f: + lines = f.readlines() + +# Counts +total_lines = len(lines) +error_count = sum(1 for line in lines if '- ERROR -' in line) +warning_count = sum(1 for line in lines if '- WARNING -' in line) +debug_count = sum(1 for line in lines if '- DEBUG -' in line) + +# Connection count +conn_count = sum(1 for line in lines if 'Connecting to server' in line) +query_count = sum(1 for line in lines if 'execute: Starting' in line) + +print(f"📈 Statistics:") +print(f" Total log lines: {total_lines:,}") +print(f" Errors: {error_count}") +print(f" Warnings: {warning_count}") +print(f" Debug messages: {debug_count:,}") +print(f" Connections: {conn_count}") +print(f" Queries: {query_count}") +print() + +# Show errors +if error_count > 0: + print(f"🚨 Errors Found ({error_count}):") + for line in lines: + if '- ERROR -' in line: + print(f" {line.strip()}") + print() + +# Show warnings +if warning_count > 0: + print(f"⚠️ Warnings Found ({warning_count}):") + for line in lines: + if '- WARNING -' in line: + print(f" {line.strip()}") + print() + +# Show first and last timestamps +if total_lines > 0: + first_line = lines[0] + last_line = lines[-1] + print(f"⏱️ Time Range:") + print(f" First: {first_line[:23]}") + print(f" Last: {last_line[:23]}") + print() + +print("=" * 70) +``` diff --git a/MSSQL-Python-Logging-Design.md b/MSSQL-Python-Logging-Design.md new file mode 100644 index 00000000..d5c41947 --- /dev/null +++ b/MSSQL-Python-Logging-Design.md @@ -0,0 +1,2315 @@ +# Simplified Logging System Design for mssql-python + +**Version:** 2.0 +**Date:** November 6, 2025 +**Status:** Design Document + +--- + +## Table of Contents + +1. [Executive Summary](#executive-summary) +2. [Design Goals](#design-goals) +3. [Architecture Overview](#architecture-overview) +4. [Component Details](#component-details) +5. [Data Flow & Workflows](#data-flow--workflows) +6. [Performance Considerations](#performance-considerations) +7. [Implementation Plan](#implementation-plan) +8. [Code Examples](#code-examples) +9. [Migration Guide](#migration-guide) +10. [Testing Strategy](#testing-strategy) +11. [Appendix](#appendix) + +--- + +## Executive Summary + +This document describes a **simplified, single-level logging system** for mssql-python that: + +- ✅ Uses **DEBUG level only** - no categorization +- ✅ Provides **all-or-nothing** logging (if enabled, see everything) +- ✅ Uses **single Python logger** with cached C++ access +- ✅ Maintains **log sequence integrity** (single writer) +- ✅ Simplifies architecture (2 components only) +- ✅ Clear performance warning (don't enable without reason) +- ✅ Future: Universal profiler for performance analysis (separate from logging) + +### Key Philosophy + +**"If you need logging, you need to see what's broken"** + +- No partial information through level filtering +- Logging is a troubleshooting tool, not a production feature +- Enable when debugging, disable otherwise +- Performance analysis will be handled by a future profiler enhancement + +### Key Differences from Previous System + +| Aspect | Previous System | New System | +| --- | --- | --- | +| **Levels** | FINE/FINER/FINEST | **DEBUG only** (all or nothing) | +| **User API** | `logger.setLevel(level)` | `setup_logging()` | +| **Philosophy** | Granular control | All or nothing - see everything or nothing | +| **Performance** | Minor overhead | Same overhead, but clearer warning | +| **Use Case** | Diagnostics at different levels | Troubleshooting only (profiler for perf) | +| **Complexity** | Multiple levels | Single level - simpler | + +--- + +## Design Goals + +### Primary Goals + +1. **Simplicity First**: Single level (DEBUG) - all or nothing +2. **Clear Purpose**: Logging is for troubleshooting, not production monitoring +3. **Performance Warning**: Explicit that logging has overhead (~2-5%) +4. **Future-Proof**: Profiler (future) handles performance analysis separately +5. **Easy to Use**: One function call: `setup_logging()` + +### Non-Goals + +- ❌ Multiple log levels (defeats "see everything" philosophy) +- ❌ Production monitoring (use proper monitoring tools) +- ❌ Performance measurement (use profiler, coming soon) +- ❌ Complex configuration files +- ❌ Async logging (synchronous is fine for diagnostics) + +--- + +## Architecture Overview + +### High-Level Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ USER CODE │ +│ │ +│ import mssql_python │ +│ │ +│ # Enable logging - see EVERYTHING │ +│ mssql_python.setup_logging() │ +│ │ +│ # Use the driver - all operations logged at DEBUG level │ +│ conn = mssql_python.connect(...) │ +│ │ +└─────────────────────────────────────────────────────────────────┘ + ↓ +┌────────────────────────────────────────────────────────────────┐ +│ PYTHON LAYER │ +│ │ +│ ┌───────────────────────────────────────────────────────┐ │ +│ │ logging.py (Single logger, DEBUG level only) │ │ +│ │ │ │ +│ │ • Single Python logger instance │ │ +│ │ • DEBUG level only (no FINE/FINER/FINEST) │ │ +│ │ • File handler with rotation │ │ +│ │ • Credential sanitization │ │ +│ │ • Thread-safe │ │ +│ │ │ │ +│ │ class MSSQLLogger: │ │ +│ │ def debug(msg): ... │ │ +│ │ def setup_logging(output, path): ... │ │ +│ │ │ │ +│ │ logger = MSSQLLogger() # Singleton │ │ +│ └───────────────────────────────────────────────────────┘ │ +│ ↑ │ +│ │ │ +│ ┌───────────────────────────────────────────────────────┐ │ +│ │ connection.py, cursor.py, etc. │ │ +│ │ │ │ +│ │ from .logging import logger │ │ +│ │ logger.debug("Connecting...") │ │ +│ │ logger.debug("Executing query: %s", sql) │ │ +│ │ logger.debug("Parameters: %s", params) │ │ +│ └───────────────────────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────────────┘ + ↑ + │ (cached import) +┌────────────────────────────────────────────────────────────────┐ +│ C++ LAYER │ +│ │ +│ ┌───────────────────────────────────────────────────────┐ │ +│ │ logger_bridge.hpp / logger_bridge.cpp │ │ +│ │ │ │ +│ │ • Caches Python logger on first use │ │ +│ │ • Caches current log level (DEBUG or OFF) │ │ +│ │ • Fast level check before ANY work │ │ +│ │ • Single macro: LOG_DEBUG() │ │ +│ │ │ │ +│ │ class LoggerBridge: │ │ +│ │ static PyObject* cached_logger │ │ +│ │ static int cached_level │ │ +│ │ static bool isLoggable() │ │ +│ │ static void log(msg) │ │ +│ └───────────────────────────────────────────────────────┘ │ +│ ↑ │ +│ │ │ +│ ┌───────────────────────────────────────────────────────┐ │ +│ │ ddbc_*.cpp (all C++ modules) │ │ +│ │ │ │ +│ │ #include "logger_bridge.hpp" │ │ +│ │ │ │ +│ │ LOG_DEBUG("Executing query: %s", sql); │ │ +│ │ LOG_DEBUG("Binding parameter: %d", param_index); │ │ +│ │ LOG_DEBUG("Fetched %d rows", row_count); │ │ +│ └───────────────────────────────────────────────────────┘ │ +└────────────────────────────────────────────────────────────────┘ + ↓ +┌────────────────────────────────────────────────────────────────┐ +│ LOG FILE │ +│ │ +│ mssql_python_logs/mssql_python_trace_20251106_143022_12345.log │ +│ │ +│ 2025-11-06 14:30:22,145 - DEBUG - connection.py:42 - │ +│ [Python] Connecting to server: localhost │ +│ 2025-11-06 14:30:22,146 - DEBUG - logger_bridge.cpp:89 - │ +│ [DDBC] Allocating connection handle │ +│ 2025-11-06 14:30:22,150 - DEBUG - cursor.py:28 - │ +│ [Python] Executing query: SELECT * FROM users │ +│ 2025-11-06 14:30:22,151 - DEBUG - cursor.py:45 - │ +│ [Python] Parameters: [42, 'test@example.com'] │ +│ 2025-11-06 14:30:22,200 - DEBUG - cursor.py:89 - │ +│ [Python] Fetched 10 rows │ +└────────────────────────────────────────────────────────────────┘ +``` + +### Component Breakdown + +| Component | File(s) | Responsibility | Lines of Code (est.) | +| --- | --- | --- | --- | +| **Python Logger** | `logging.py` | Core logger, levels, handlers | ~200 | +| **C++ Bridge** | `logger_bridge.hpp/.cpp` | Cached Python access, macros | ~150 | +| **Pybind Glue** | `bindings.cpp` (update) | Expose sync functions | ~30 | +| **Python Usage** | `connection.py`, etc. | Use logger in Python code | Varies | +| **C++ Usage** | `ddbc_*.cpp` | Use LOG_* macros | Varies | + +**Total New Code: ~380 lines** + +--- + +## Component Details + +### Component 1: Python Logger (`logging.py`) + +#### Purpose +Single source of truth for all logging. Provides Driver Levels and manages file output. + +#### Key Responsibilities +1. Define custom log levels (FINE/FINER/FINEST) +2. Setup rotating file handler +3. Provide convenience methods (`fine()`, `finer()`, `finest()`) +4. Sanitize sensitive data (passwords, tokens) +5. Synchronize level changes with C++ +6. Thread-safe operation + +#### Design Details + +**Singleton Pattern** +- One instance per process +- Thread-safe initialization +- Lazy initialization on first import + +**Custom Log Levels** +```python +# Driver Levels (Primary API - Recommended) +FINEST = 5 # Ultra-detailed trace (most verbose) +FINER = 15 # Detailed diagnostics +FINE = 18 # Standard diagnostics (recommended default) + +# Python Standard Levels (Also Available - For Compatibility) +# DEBUG = 10 # Python standard debug level +# INFO = 20 # Python standard info level +# WARNING = 30 # Python standard warning level +# ERROR = 40 # Python standard error level +# CRITICAL = 50 # Python standard critical level +``` + +**Output Destination Constants** +```python +# Output destinations (flat namespace, like log levels) +FILE = 'file' # Log to file only (default) +STDOUT = 'stdout' # Log to stdout only +BOTH = 'both' # Log to both file and stdout +``` + +**Why these numbers?** +- Driver Levels (FINEST/FINER/FINE) are **recommended** for driver diagnostics +- Standard Python levels (DEBUG/INFO/WARNING/ERROR) also work for compatibility +- FINE=18 < INFO=20, so FINE level includes INFO and above +- Higher number = higher priority (standard convention) + +**File Handler Configuration** +- **Location**: `./mssql_python_logs/` folder (created automatically if doesn't exist) +- **Naming**: `mssql_python_trace_YYYYMMDDHHMMSS_PID.log` (timestamp with no separators) +- **Custom Path**: Users can specify via `log_file_path` parameter (creates parent directories if needed) +- **Rotation**: 512MB max, 5 backup files +- **Format**: Comma-separated fields: `Timestamp, ThreadID, Level, Location, Source, Message` (importable as CSV) +- **Header**: File includes metadata header with PID, script name, Python version, driver version, start time, OS info + +**Output Handler Configuration** +- **Default**: File only (using `FILE` constant) +- **File Handler**: RotatingFileHandler with 512MB max, 5 backup files +- **Stdout Handler**: StreamHandler to sys.stdout (optional) +- **Both Mode**: Adds both file and stdout handlers simultaneously +- **Format**: Same format for both file and stdout handlers + +**Thread Tracking System** + +The logging system uses **OS native thread IDs** to track operations across multi-threaded applications. + +**Use Cases:** +- Multi-threaded applications with multiple concurrent connections +- Connection pooling scenarios (track which thread handles which connection) +- Multiple cursors per connection (distinguish operations by thread) +- Performance profiling (measure operation duration per thread) +- Debugger correlation (thread IDs match debugger thread views) +- Production debugging (filter logs by specific thread) + +**Design:** + +1. **OS Native Thread ID** + - Uses `threading.get_native_id()` (Python 3.8+) + - Returns OS-level thread identifier + - Matches thread IDs shown in debuggers (Visual Studio, gdb, lldb) + - Compatible with system monitoring tools + - Thread-safe, no locks required + +2. **Log Format:** + ``` + Timestamp, ThreadID, Level, Location, Source, Message + 2025-11-06 10:30:15.100, 8581947520, DEBUG, connection.py:156, Python, Allocating environment handle + 2025-11-06 10:30:15.101, 8581947520, DEBUG, connection.cpp:22, DDBC, Allocating ODBC environment handle + 2025-11-06 10:30:15.200, 8582001664, DEBUG, connection.py:42, Python, Different thread operation + ``` + + **Structure:** + - ThreadID for filtering by thread + - Source distinguishes Python vs DDBC (C++) operations + - Location shows exact file:line + - Timestamp with milliseconds + - Comma-separated fields (importable as CSV if needed) + +3. **Automatic Injection:** + - Custom `logging.Filter` adds thread_id to LogRecord using `threading.get_native_id()` + - CSVFormatter extracts Source from message prefix `[Python]` or `[DDBC]` + - No manual thread ID passing required + +4. **Implementation Components:** + ```python + import threading + import logging + + class TraceIDFilter(logging.Filter): + """Adds OS native thread ID to log records""" + def filter(self, record): + record.trace_id = threading.get_native_id() + return True + + class CSVFormatter(logging.Formatter): + """Formats logs with structured fields""" + def format(self, record): + # Extract source from message prefix [Python] or [DDBC] + source = 'Python' + message = record.getMessage() + if message.startswith('[DDBC]'): + source = 'DDBC' + message = message[7:].strip() + elif message.startswith('[Python]'): + source = 'Python' + message = message[9:].strip() + + # Format as CSV + timestamp = self.formatTime(record, '%Y-%m-%d %H:%M:%S') + ms = f"{record.msecs:03.0f}" + location = f"{record.filename}:{record.lineno}" + thread_id = getattr(record, 'trace_id', '-') + + return f"{timestamp}.{ms}, {thread_id}, {record.levelname}, {location}, {source}, {message}" + ``` + +5. **File Header:** + ```python + def _write_log_header(self): + \"\"\"Write metadata header to log file\"\"\" + with open(self.log_file_path, 'w') as f: + f.write(f\"# MSSQL-Python Driver Log | \" + f\"Script: {os.path.basename(sys.argv[0])} | \" + f\"PID: {os.getpid()} | \" + f\"Log Level: DEBUG | \" + f\"Python: {sys.version.split()[0]} | \" + f\"Driver: {driver_version} | \" + f\"Start: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} | \" + f\"OS: {platform.platform()}\\n\")\n f.write(\"Timestamp, ThreadID, Level, Location, Source, Message\\n\")\n ``` + +6. **Thread Safety:** + - `threading.get_native_id()` is thread-safe + - Each thread gets its own unique OS-level ID + - No locks needed for thread ID access + - CSV formatter is stateless and thread-safe + +7. **Performance:** + - Zero overhead when logging disabled + - Minimal overhead when enabled (~1 μs per log call) + - CSV formatting is simple string concatenation + - No complex parsing or regex operations + +**Example Log Output:** +``` +# MSSQL-Python Driver Log | Script: main.py | PID: 80677 | Log Level: DEBUG | Python: 3.13.7 | Driver: unknown | Start: 2025-11-06 20:40:11 | OS: macOS-26.1-arm64-arm-64bit-Mach-O +Timestamp, ThreadID, Level, Location, Source, Message +2025-11-06 20:42:39.704, 1347850, DEBUG, connection.cpp:22, DDBC, Allocating ODBC environment handle +2025-11-06 20:42:39.705, 1347850, DEBUG, connection.py:156, Python, Connection opened +2025-11-06 20:42:39.706, 1347850, DEBUG, cursor.py:28, Python, Cursor created +2025-11-06 20:42:39.707, 1347850, DEBUG, cursor.py:89, Python, Executing query: SELECT * FROM users +2025-11-06 20:42:39.710, 1347850, DEBUG, cursor.py:145, Python, Fetched 42 rows +2025-11-06 20:42:39.711, 1347850, DEBUG, connection.py:234, Python, Connection closed +``` + + + + +**Multi-Threaded Example:** +``` +# Thread 8581947520 logs: +2025-11-06 10:30:15.100, 8581947520, DEBUG, connection.py:156, Python, Connection opened +2025-11-06 10:30:15.102, 8581947520, DEBUG, cursor.py:28, Python, Cursor created +2025-11-06 10:30:15.103, 8581947520, DEBUG, cursor.py:89, Python, Query: SELECT * FROM users +2025-11-06 10:30:15.105, 8581947520, DEBUG, cursor.py:145, Python, Fetched 100 rows + +# Thread 8582001664 logs (interleaved, but distinguishable by ThreadID): +2025-11-06 10:30:15.104, 8582001664, DEBUG, connection.py:156, Python, Connection opened +2025-11-06 10:30:15.106, 8582001664, DEBUG, cursor.py:89, Python, Query: SELECT * FROM orders +2025-11-06 10:30:15.108, 8582001664, DEBUG, cursor.py:145, Python, Fetched 50 rows +``` + + +**Hybrid API Approach** + +The logger supports both Driver Levels and Python standard logging levels: + +1. **Driver Levels (FINE/FINER/FINEST)** - **Recommended** + - Use in driver internal code (connection.py, cursor.py, etc.) + - Provides granular control specific to database operations + - Inspired by proven enterprise logging patterns + - Clear semantic meaning for database diagnostics + +2. **Python Standard Levels (DEBUG/INFO/WARNING/ERROR)** - **Compatible** + - Available for users familiar with Python logging + - Works seamlessly alongside Driver levels + - Good for application-level code using the driver + - No learning curve for Python developers + +**When to Use Which:** +- **Driver internals**: Prefer `logger.fine()`, `logger.finer()`, `logger.finest()` +- **Application code**: Either style works; use what's familiar +- **Error logging**: `logger.error()` or `logger.critical()` work well (Python standard) +- **Production**: Set `logger.setLevel(CRITICAL)` to minimize overhead + +**🔑 KEY COMPATIBILITY GUARANTEE:** + +**Existing code using Python standard levels will continue to work when Driver Levels are enabled!** + +```python +# User's existing code (Python standard levels) +logger.info("Connected to database") +logger.warning("Query took 5 seconds") +logger.error("Connection timeout") + +# Enable driver diagnostics with Driver Levels +logger.setLevel(FINE) # FINE = 18 + +# ✅ Result: ALL messages above will appear in logs! +# Because: INFO (20), WARNING (30), ERROR (40) are all > FINE (18) +# The level hierarchy ensures backward compatibility +``` + +**Level Filtering Rules:** +- `setLevel(FINE)` (18) → Shows: FINE, INFO, WARNING, ERROR, CRITICAL +- `setLevel(FINER)` (15) → Shows: FINER, FINE, INFO, WARNING, ERROR, CRITICAL +- `setLevel(FINEST)` (5) → Shows: Everything (all levels) +- `setLevel(logging.INFO)` (20) → Shows: INFO, WARNING, ERROR, CRITICAL (hides FINE/FINER/FINEST) + +#### Public API + +```python +from mssql_python.logging import logger, FINE, FINER, FINEST, FILE, STDOUT, BOTH + +# Driver Levels API (Recommended for mssql-python) +# ================================================= + +# Check if level enabled +if logger.isEnabledFor(FINER): + expensive_data = compute_diagnostics() + logger.finer(f"Diagnostics: {expensive_data}") + +# Log at Driver Levels (recommended) +logger.fine("Standard diagnostic message") # Primary diagnostic level +logger.finer("Detailed diagnostic message") # Detailed troubleshooting +logger.finest("Ultra-detailed trace message") # Deep debugging + +# Change level with Driver Level constants (recommended) +logger.setLevel(FINE) # Standard diagnostics +logger.setLevel(FINER) # Detailed diagnostics +logger.setLevel(FINEST) # Ultra-detailed (all diagnostics) +logger.setLevel(CRITICAL) # Errors only (production) + +# Configure output destination +logger.output = FILE # File only (default) +logger.output = STDOUT # Stdout only +logger.output = BOTH # Both file and stdout + +# Or set output when setting level +logger.setLevel(FINE, output=BOTH) + +# Custom log file path +logger.setLevel(FINE, log_file_path="/var/log/myapp.log") +logger.setLevel(FINE, output=BOTH, log_file_path="/tmp/debug.log") + +# Python Standard API (Also Available for Compatibility) +# ====================================================== +import logging + +# Also works - standard Python logging methods +logger.info("Informational message") # Python standard +logger.warning("Warning message") # Python standard +logger.error("Error message") # Python standard +logger.debug("Debug message") # Python standard + +# Can also use Python standard level constants +logger.setLevel(logging.DEBUG) # Python standard +logger.setLevel(logging.INFO) # Python standard + +# Get log file location +print(f"Logging to: {logger.log_file}") +``` + +#### Internal Structure + +```python +class MSSQLLogger: + _instance = None + _lock = threading.Lock() + + def __init__(self): + self._logger = logging.getLogger('mssql_python') + self._logger.setLevel(logging.CRITICAL) # OFF by default + self._output_mode = FILE # Default to file only + self._file_handler = None + self._stdout_handler = None + self._custom_log_path = None # Custom log file path (optional) + self._setup_handlers() + self._trace_counter = 0 + self._trace_lock = threading.Lock() + + # Trace ID support (contextvars for automatic propagation) + import contextvars + self._trace_id_var = contextvars.ContextVar('trace_id', default=None) + + # Add trace ID filter to logger + self._logger.addFilter(self._TraceIDFilter(self._trace_id_var)) + + class _TraceIDFilter(logging.Filter): + """Filter that adds trace_id to log records""" + def __init__(self, trace_id_var): + super().__init__() + self._trace_id_var = trace_id_var + + def filter(self, record): + trace_id = self._trace_id_var.get() + record.trace_id = trace_id if trace_id else '-' + return True + + def _setup_handlers(self): + # Setup handlers based on output mode + # File handler: RotatingFileHandler + # Stdout handler: StreamHandler(sys.stdout) + pass + + def _reconfigure_handlers(self): + # Remove existing handlers and add new ones based on output mode + pass + + @property + def output(self): + return self._output_mode + + @output.setter + def output(self, mode): + # Validate mode and reconfigure handlers + if mode not in (FILE, STDOUT, BOTH): + raise ValueError(f"Invalid output mode: {mode}") + self._output_mode = mode + self._reconfigure_handlers() + + def _sanitize_message(self, msg: str) -> str: + # Remove PWD=..., Password=..., etc. + pass + + def generate_trace_id(self, prefix: str = "TRACE") -> str: + """Generate unique trace ID: PREFIX-PID-ThreadID-Counter""" + with self._trace_lock: + self._trace_counter += 1 + counter = self._trace_counter + + pid = os.getpid() + thread_id = threading.get_ident() + return f"{prefix}-{pid}-{thread_id}-{counter}" + + def set_trace_id(self, trace_id: str): + """Set trace ID for current context (auto-propagates to child contexts)""" + self._trace_id_var.set(trace_id) + + def get_trace_id(self) -> Optional[str]: + """Get current trace ID (None if not set)""" + return self._trace_id_var.get() + + def clear_trace_id(self): + """Clear trace ID for current context""" + self._trace_id_var.set(None) + + def _notify_cpp_level_change(self): + # Call C++ to update cached level + pass + + # Public methods: fine(), finer(), finest(), etc. +``` + +--- + +### Component 2: C++ Logger Bridge + +#### Purpose +Provide high-performance logging from C++ with zero overhead when disabled. + +#### Key Responsibilities +1. Cache Python logger object (import once) +2. Cache current log level (check fast) +3. Provide fast `isLoggable()` check +4. Format messages only when needed +5. Call Python logger only when enabled +6. Thread-safe operation + +#### Design Details + +**Caching Strategy** + +```cpp +class LoggerBridge { +private: + // Cached Python objects (imported once) + static PyObject* cached_logger_; + static PyObject* fine_method_; + static PyObject* finer_method_; + static PyObject* finest_method_; + + // Cached log level (synchronized from Python) + static std::atomic cached_level_; + + // Thread safety + static std::mutex mutex_; + static bool initialized_; + + // Private constructor (singleton) + LoggerBridge() = default; + +public: + // Initialize (called once from Python) + static void initialize(); + + // Update level when Python calls setLevel() + static void updateLevel(int level); + + // Fast level check (inline, zero overhead) + static inline bool isLoggable(int level) { + return level >= cached_level_.load(std::memory_order_relaxed); + } + + // Log a message (only called if isLoggable() returns true) + static void log(int level, const char* file, int line, + const char* format, ...); +}; +``` + +**Performance Optimizations** + +1. **Atomic Level Check**: `std::atomic` for lock-free reads +2. **Early Exit**: `if (!isLoggable(level)) return;` before any work +3. **Lazy Formatting**: Only format strings if logging enabled +4. **Cached Methods**: Import Python methods once, reuse forever +5. **Stack Buffers**: Use stack allocation for messages (4KB default) + +**Macro API** + +```cpp +// Convenience macros for use throughout C++ code +#define LOG_FINE(fmt, ...) \ + do { \ + if (mssql_python::logging::LoggerBridge::isLoggable(25)) { \ + mssql_python::logging::LoggerBridge::log(25, __FILE__, __LINE__, fmt, ##__VA_ARGS__); \ + } \ + } while(0) + +#define LOG_FINER(fmt, ...) \ + do { \ + if (mssql_python::logging::LoggerBridge::isLoggable(15)) { \ + mssql_python::logging::LoggerBridge::log(15, __FILE__, __LINE__, fmt, ##__VA_ARGS__); \ + } \ + } while(0) + +#define LOG_FINEST(fmt, ...) \ + do { \ + if (mssql_python::logging::LoggerBridge::isLoggable(5)) { \ + mssql_python::logging::LoggerBridge::log(5, __FILE__, __LINE__, fmt, ##__VA_ARGS__); \ + } \ + } while(0) +``` + +**Why Macros?** +- Include `__FILE__` and `__LINE__` automatically +- Inline the `isLoggable()` check for zero overhead +- Cleaner call sites: `LOG_FINE("msg")` vs `LoggerBridge::log(FINE, __FILE__, __LINE__, "msg")` + +#### Thread Safety + +**Problem**: Multiple C++ threads logging simultaneously +**Solution**: Lock only during Python call, not during level check + +```cpp +void LoggerBridge::log(int level, const char* file, int line, + const char* format, ...) { + // Fast check without lock + if (!isLoggable(level)) return; + + // Format message on stack (no allocation) + char buffer[4096]; + va_list args; + va_start(args, format); + vsnprintf(buffer, sizeof(buffer), format, args); + va_end(args); + + // Lock only for Python call + std::lock_guard lock(mutex_); + + // Acquire GIL and call Python + PyGILState_STATE gstate = PyGILState_Ensure(); + PyObject* result = PyObject_CallMethod( + cached_logger_, "log", "is", level, buffer + ); + Py_XDECREF(result); + PyGILState_Release(gstate); +} +``` + +#### Initialization Flow + +```cpp +// Called from Python during module import +void LoggerBridge::initialize() { + std::lock_guard lock(mutex_); + + if (initialized_) return; + + // Import Python logger module + PyObject* logging_module = PyImport_ImportModule("mssql_python.logging"); + if (!logging_module) { + // Handle error + return; + } + + // Get logger instance + cached_logger_ = PyObject_GetAttrString(logging_module, "logger"); + Py_DECREF(logging_module); + + if (!cached_logger_) { + // Handle error + return; + } + + // Cache methods for faster calls + fine_method_ = PyObject_GetAttrString(cached_logger_, "fine"); + finer_method_ = PyObject_GetAttrString(cached_logger_, "finer"); + finest_method_ = PyObject_GetAttrString(cached_logger_, "finest"); + + // Get initial level + PyObject* level_obj = PyObject_GetAttrString(cached_logger_, "level"); + if (level_obj) { + cached_level_.store(PyLong_AsLong(level_obj)); + Py_DECREF(level_obj); + } + + initialized_ = true; +} +``` + +--- + +## Data Flow & Workflows + +### Workflow 1: User Enables Logging + +``` +┌─────────────────────────────────────────────────────────┐ +│ User Code │ +│ │ +│ logger.setLevel(FINE) │ +│ │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ logging.py: MSSQLLogger.setLevel() │ +│ │ +│ 1. Update Python logger level │ +│ self._logger.setLevel(FINE) │ +│ │ +│ 2. Notify C++ bridge │ +│ ddbc_bindings.update_log_level(FINE) │ +│ │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ C++: LoggerBridge::updateLevel() │ +│ │ +│ cached_level_.store(FINE) │ +│ // Atomic update, visible to all │ +│ // C++ threads immediately │ +│ │ +└─────────────────────────────────────────────────────────┘ + │ + ↓ + [Logging now enabled at FINE level] +``` + +**Time Complexity**: O(1) +**Thread Safety**: Atomic store, lock-free for readers + +**Level Hierarchy** (lower number = more verbose): +``` +FINEST (5) ← Driver Levels: Ultra-detailed +DEBUG (10) ← Python standard +FINER (15) ← Driver Levels: Detailed +FINE (18) ← Driver Levels: Standard (default for troubleshooting) +INFO (20) ← Python standard +WARNING (30) ← Python standard +ERROR (40) ← Python standard +CRITICAL (50) ← Python standard + +Example: Setting FINE (18) will show: + ✓ FINE (18), INFO (20), WARNING (30), ERROR (40), CRITICAL (50) + ✗ FINER (15), DEBUG (10), FINEST (5) - too verbose, filtered out +``` + +**⚠️ IMPORTANT - Backward Compatibility:** + +When you enable Driver Levels with `logger.setLevel(FINE)`, **all Python standard levels that are higher than FINE will still appear in logs:** + +| Your Code Uses | Will Appear at FINE? | Will Appear at FINER? | Will Appear at FINEST? | +|----------------|---------------------|----------------------|------------------------| +| `logger.finest()` (5) | ❌ No (5 < 18) | ❌ No (5 < 15) | ✅ Yes (5 ≥ 5) | +| `logger.debug()` (10) | ❌ No (10 < 18) | ❌ No (10 < 15) | ✅ Yes (10 ≥ 5) | +| `logger.finer()` (15) | ❌ No (15 < 18) | ✅ Yes (15 ≥ 15) | ✅ Yes (15 ≥ 5) | +| `logger.fine()` (18) | ✅ Yes (18 ≥ 18) | ✅ Yes (18 ≥ 15) | ✅ Yes (18 ≥ 5) | +| `logger.info()` (20) | ✅ **Yes** (20 ≥ 18) | ✅ **Yes** (20 ≥ 15) | ✅ **Yes** (20 ≥ 5) | +| `logger.warning()` (30) | ✅ **Yes** (30 ≥ 18) | ✅ **Yes** (30 ≥ 15) | ✅ **Yes** (30 ≥ 5) | +| `logger.error()` (40) | ✅ **Yes** (40 ≥ 18) | ✅ **Yes** (40 ≥ 15) | ✅ **Yes** (40 ≥ 5) | +| `logger.critical()` (50) | ✅ **Yes** (50 ≥ 18) | ✅ **Yes** (50 ≥ 15) | ✅ **Yes** (50 ≥ 5) | + +**Bottom Line:** Existing code using `info()`, `warning()`, `error()` continues to work! No migration needed! 🎉 + +--- + +### Workflow 2: Python Code Logs a Message + +``` +┌─────────────────────────────────────────────────────────┐ +│ connection.py │ +│ │ +│ logger.fine("Connecting to server") │ +│ │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ logging.py: MSSQLLogger.fine() │ +│ │ +│ 1. Check if enabled (fast) │ +│ if not isEnabledFor(FINE): │ +│ return │ +│ │ +│ 2. Add prefix │ +│ msg = f"[Python] {msg}" │ +│ │ +│ 3. Sanitize (if needed) │ +│ msg = sanitize(msg) │ +│ │ +│ 4. Log via Python's logger │ +│ self._logger.log(FINE, msg) │ +│ │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ Python logging.Logger │ +│ │ +│ 1. Format message with timestamp │ +│ 2. Write to file handler │ +│ 3. Rotate if needed │ +│ │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ Log File │ +│ │ +│ 2025-10-31 14:30:22,145 - FINE - │ +│ connection.py:42 - [Python] │ +│ Connecting to server │ +└─────────────────────────────────────────────────────────┘ +``` + +**Time Complexity**: O(1) for check, O(log n) for file I/O +**When Disabled**: Single `if` check, immediate return + +--- + +### Workflow 3: C++ Code Logs a Message (Logging Enabled) + +``` +┌─────────────────────────────────────────────────────────┐ +│ ddbc_connection.cpp │ +│ │ +│ LOG_FINE("Allocating handle: %p", handle) │ +│ │ +└────────────────────────┬────────────────────────────────┘ + │ (macro expands to:) + ↓ +┌─────────────────────────────────────────────────────────┐ +│ Expanded Macro │ +│ │ +│ if (LoggerBridge::isLoggable(FINE)) { │ +│ LoggerBridge::log(FINE, __FILE__, __LINE__, │ +│ "Allocating handle: %p", handle); │ +│ } │ +│ │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ C++: LoggerBridge::isLoggable() │ +│ │ +│ return FINE >= cached_level_; │ +│ // Inline, lock-free, ~1 CPU cycle │ +│ │ +│ Result: TRUE (logging enabled) │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ C++: LoggerBridge::log() │ +│ │ +│ 1. Format message with vsnprintf │ +│ buffer = "Allocating handle: 0x7fff1234 │ +│ [file.cpp:42]" │ +│ │ +│ 2. Acquire mutex + GIL │ +│ │ +│ 3. Call Python logger │ +│ cached_logger_.log(FINE, buffer) │ +│ │ +│ 4. Release GIL + mutex │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ Python: logger.log() │ +│ │ +│ (Same as Python workflow) │ +│ - Add [DDBC] prefix │ +│ - Sanitize │ +│ - Write to file │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ Log File │ +│ │ +│ 2025-10-31 14:30:22,146 - FINE - │ +│ logger_bridge.cpp:89 - [DDBC] │ +│ Allocating handle: 0x7fff1234 [file.cpp:42] │ +│ │ +└─────────────────────────────────────────────────────────┘ +``` + +**Time Complexity**: +- Level check: O(1), ~1 CPU cycle +- Message formatting: O(n) where n = message length +- Python call: O(1) + GIL acquisition overhead +- File I/O: O(log n) + +--- + +### Workflow 4: C++ Code Logs a Message (Logging Disabled) + +``` +┌─────────────────────────────────────────────────────────┐ +│ ddbc_connection.cpp │ +│ │ +│ LOG_FINE("Allocating handle: %p", handle) │ +│ │ +└────────────────────────┬────────────────────────────────┘ + │ (macro expands to:) + ↓ +┌─────────────────────────────────────────────────────────┐ +│ Expanded Macro │ +│ │ +│ if (LoggerBridge::isLoggable(FINE)) { │ +│ // ... logging code ... │ +│ } │ +│ │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ +┌─────────────────────────────────────────────────────────┐ +│ C++: LoggerBridge::isLoggable() │ +│ │ +│ return FINE >= cached_level_; │ +│ // cached_level_ = CRITICAL (50) │ +│ // FINE (25) < CRITICAL (50) │ +│ │ +│ Result: FALSE │ +└────────────────────────┬────────────────────────────────┘ + │ + ↓ + [DONE - No further work] + [Zero overhead - just one if check] +``` + +**Time Complexity**: O(1), ~1 CPU cycle +**Overhead**: Single comparison instruction +**No**: Formatting, Python calls, GIL acquisition, I/O + +--- + +### Workflow 5: Conditional Expensive Logging + +For operations that are expensive to compute: + +```cpp +// In ddbc_query.cpp + +// Quick operation - always use macro +LOG_FINE("Executing query: %s", sanitized_sql); + +// Expensive operation - manual check first +if (LoggerBridge::isLoggable(FINEST)) { + // Only compute if FINEST enabled + std::string full_diagnostics = generateFullDiagnostics(); + std::string memory_stats = getMemoryStatistics(); + std::string connection_pool = dumpConnectionPool(); + + LOG_FINEST("Full diagnostics:\n%s\n%s\n%s", + full_diagnostics.c_str(), + memory_stats.c_str(), + connection_pool.c_str()); +} +``` + +**Pattern**: +1. Use `LOG_*` macros for cheap operations +2. For expensive operations: + - Check `isLoggable()` first + - Compute expensive data only if true + - Then call `LOG_*` macro + +--- + +## Performance Considerations + +### Performance Goals + +| Scenario | Target Overhead | Achieved | +| --- | --- | --- | +| Logging disabled | < 0.1% | ~1 CPU cycle per log call | +| Logging enabled (FINE) | < 5% | ~2-4% (mostly I/O) | +| Logging enabled (FINEST) | < 10% | ~5-8% (more messages) | + +### Bottleneck Analysis + +**When Logging Disabled** ✅ +- **Bottleneck**: None +- **Cost**: Single atomic load + comparison +- **Optimization**: Inline check, branch predictor optimizes away + +**When Logging Enabled** ⚠️ +- **Bottleneck 1**: String formatting (`vsnprintf`) + - **Cost**: ~1-5 μs per message + - **Mitigation**: Only format if isLoggable() + +- **Bottleneck 2**: GIL acquisition + - **Cost**: ~0.5-2 μs per call + - **Mitigation**: Minimize Python calls, batch if possible + +- **Bottleneck 3**: File I/O + - **Cost**: ~10-100 μs per write + - **Mitigation**: Python's logging buffers internally + +### Memory Considerations + +**Stack Usage** +- Message buffer: 4KB per log call (stack-allocated) +- Safe for typical messages (<4KB) +- Long messages truncated (better than overflow) + +**Heap Usage** +- Cached Python objects: ~200 bytes (one-time) +- Python logger internals: ~2KB (managed by Python) +- File buffers: ~8KB (Python's logging) + +**Total**: ~10KB steady-state overhead + +### Threading Implications + +**C++ Threading** +- `isLoggable()`: Lock-free, atomic read +- `log()`: Mutex only during Python call +- Multiple threads can check level simultaneously +- Serialized only for actual logging + +**Python Threading** +- GIL naturally serializes Python logging calls +- File handler has internal locking +- No additional synchronization needed + +**Recommendation**: Safe for multi-threaded applications + +--- + +## Implementation Plan + +### Phase 1: Core Infrastructure (Week 1) + +**Tasks**: +1. ✅ Create `logging.py` + - Custom levels (FINE/FINER/FINEST) + - Singleton MSSQLLogger class + - File handler setup + - Basic methods (fine, finer, finest) + - Sanitization logic + +2. ✅ Create C++ bridge + - `logger_bridge.hpp` with macros + - `logger_bridge.cpp` implementation + - Caching mechanism + - Level synchronization + +3. ✅ Update pybind11 bindings + - Expose `update_log_level()` to Python + - Call `LoggerBridge::initialize()` on import + +**Deliverables**: +- `logging.py` (~200 lines) +- `logger_bridge.hpp` (~100 lines) +- `logger_bridge.cpp` (~150 lines) +- Updated `bindings.cpp` (~30 lines) + +**Testing**: +- Unit tests for Python logger +- Unit tests for C++ bridge +- Integration test: Python → C++ → Python roundtrip + +--- + +### Phase 2: Integration & Migration (Week 2) + +**Tasks**: +1. ✅ Replace `logging_config.py` with `logging.py` + - Update imports throughout codebase + - Migrate `setup_logging()` calls to `logger.setLevel()` + - Update documentation + +2. ✅ Update Python code to use new logger + - `connection.py`: Add FINE/FINER logging + - `cursor.py`: Add FINE/FINER logging + - `auth.py`, `pooling.py`: Add diagnostic logging + +3. ✅ Update C++ code to use bridge + - Add `#include "logger_bridge.hpp"` to all modules + - Replace existing logging with `LOG_*` macros + - Add conditional checks for expensive operations + +**Deliverables**: +- All Python files updated +- All C++ files updated +- Deprecated `logging_config.py` removed + +**Testing**: +- Regression tests (ensure no functionality broken) +- Performance benchmarks (compare before/after) +- Manual testing of all logging levels + +--- + +### Phase 3: Polish & Documentation (Week 3) + +**Tasks**: +1. ✅ Performance tuning + - Profile logging overhead + - Optimize hot paths + - Verify zero-overhead when disabled + +2. ✅ Documentation + - Update user guide + - Add examples for each level + - Document best practices + - Create troubleshooting guide + +3. ✅ Enhanced features + - Trace ID generation + - Connection/cursor tracking + - Query performance logging + +**Deliverables**: +- Performance report +- Updated user documentation +- Enhanced logging features + +**Testing**: +- Performance benchmarks +- Documentation review +- User acceptance testing + +--- + +### Phase 4: Release (Week 4) + +**Tasks**: +1. ✅ Final testing + - Full regression suite + - Performance validation + - Cross-platform testing (Windows, Linux, macOS) + +2. ✅ Release preparation + - Update CHANGELOG + - Update version number + - Create migration guide for users + +3. ✅ Rollout + - Merge to main branch + - Tag release + - Publish documentation + +**Deliverables**: +- Release candidate +- Migration guide +- Updated documentation + +--- + +## Code Examples + +### Example 1: Minimal Usage + +```python +""" +Minimal example - just enable driver diagnostics +""" +import mssql_python + +# Enable driver diagnostics (one line) +mssql_python.setup_logging() + +# Use the driver - all internals are now logged +conn = mssql_python.connect("Server=localhost;Database=test") +cursor = conn.cursor() +cursor.execute("SELECT 1") +conn.close() + +# That's it! Logs are in ./mssql_python_logs/mssql_python_trace_*.log +``` + +### Example 2: With Output Control + +```python +""" +Control output destination +""" +import mssql_python + +# Option 1: File only (default) +mssql_python.setup_logging() + +# Option 2: Stdout only (for CI/CD) +mssql_python.setup_logging(output='stdout') + +# Option 3: Both file and stdout (for development) +mssql_python.setup_logging(output='both') + +# Use the driver normally +connection_string = ( + "Server=myserver.database.windows.net;" + "Database=mydb;" + "UID=admin;" + "PWD=secret123;" + "Encrypt=yes;" +) + +# All operations are now logged +conn = mssql_python.connect(connection_string) +cursor = conn.cursor() +cursor.execute("SELECT * FROM users WHERE active = 1") +rows = cursor.fetchall() + +print(f"Fetched {len(rows)} rows") +conn.close() + +# Check the log file for detailed diagnostics +# Passwords will be automatically sanitized in logs +``` + +**Expected Log Output**: +``` +# MSSQL-Python Driver Log | Script: app.py | PID: 12345 | Log Level: DEBUG | Python: 3.13.7 | Start: 2025-11-06 14:30:22 +Timestamp, ThreadID, Level, Location, Source, Message +2025-11-06 14:30:22.100, 8581947520, DEBUG, connection.py:42, Python, Initializing connection +2025-11-06 14:30:22.101, 8581947520, DEBUG, connection.py:56, Python, Connection string: Server=myserver.database.windows.net;Database=mydb;UID=admin;PWD=***;Encrypt=yes; +2025-11-06 14:30:22.105, 8581947520, DEBUG, connection.cpp:123, DDBC, Allocating connection handle +2025-11-06 14:30:22.110, 8581947520, DEBUG, connection.cpp:145, DDBC, Connection established +2025-11-06 14:30:22.115, 8581947520, DEBUG, cursor.py:28, Python, Creating cursor +2025-11-06 14:30:22.120, 8581947520, DEBUG, statement.cpp:67, DDBC, Allocating statement handle +2025-11-06 14:30:22.125, 8581947520, DEBUG, cursor.py:89, Python, Executing query: SELECT * FROM users WHERE active = 1 +2025-11-06 14:30:22.130, 8581947520, DEBUG, statement.cpp:234, DDBC, SQLExecDirect called +2025-11-06 14:30:22.250, 8581947520, DEBUG, statement.cpp:267, DDBC, Query completed, rows affected: 42 +2025-11-06 14:30:22.255, 8581947520, DEBUG, cursor.py:145, Python, Fetching results +2025-11-06 14:30:22.350, 8581947520, DEBUG, cursor.py:178, Python, Fetched 42 rows +2025-11-06 14:30:22.355, 8581947520, DEBUG, connection.py:234, Python, Closing connection +``` + +--- + +### Example 3: Integrate with Your Application Logging + +```python +""" +Extensibility - plug driver logging into your application's logger +""" +import logging +import mssql_python +from mssql_python import logging as mssql_logging + +# Setup your application's logging +app_logger = logging.getLogger('myapp') +app_logger.setLevel(logging.INFO) + +# Add console handler to your logger +console = logging.StreamHandler() +console.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(message)s')) +app_logger.addHandler(console) + +# Now plug mssql-python logger into your logging +mssql_driver_logger = mssql_logging.logger # Get driver's logger instance +mssql_driver_logger.addHandler(console) # Same handler as your app +mssql_driver_logger.setLevel(mssql_logging.FINE) + +# Both your app and driver logs go to same destination +app_logger.info("Starting application") +conn = mssql_python.connect("Server=localhost;Database=test") +app_logger.info("Database connected") + +# Output shows both application and driver logs: +# 2025-11-03 10:15:22 - myapp - Starting application +# 2025-11-03 10:15:22 - mssql_python - [Python] Connecting to server +# 2025-11-03 10:15:22 - mssql_python - [Python] Connection established +# 2025-11-03 10:15:22 - myapp - Database connected +``` + +--- + +### Example 4: Mixed Driver Levels and Python Standard Levels (Backward Compatibility) + +```python +""" +Example showing existing Python standard logging code works seamlessly +when Driver Levels are enabled - NO CODE CHANGES NEEDED! +""" +import mssql_python +from mssql_python.logging import logger, FINE, FINER, FINEST +import logging + +# =================================================================== +# SCENARIO: User has existing code with Python standard levels +# =================================================================== + +class DatabaseManager: + """Existing user code using Python standard logging""" + + def connect(self, connection_string): + # User's existing code - uses INFO (level 20) + logger.info("Attempting database connection...") + + try: + conn = mssql_python.connect(connection_string) + # User's existing code - uses INFO (level 20) + logger.info("Successfully connected to database") + return conn + except Exception as e: + # User's existing code - uses ERROR (level 40) + logger.error(f"Connection failed: {e}") + raise + + def execute_query(self, conn, sql): + # User's existing code - uses WARNING (level 30) + if len(sql) > 1000: + logger.warning("Query is very long, may impact performance") + + cursor = conn.cursor() + cursor.execute(sql) + return cursor.fetchall() + +# =================================================================== +# USER ENABLES DRIVER LEVELS DIAGNOSTICS (NO CHANGES TO CODE ABOVE!) +# =================================================================== + +logger.setLevel(FINE) # FINE = 18, enables driver diagnostics + +# Now run the existing code +db = DatabaseManager() +conn = db.connect("Server=localhost;Database=test;...") +results = db.execute_query(conn, "SELECT * FROM users") + +# =================================================================== +# RESULT: ALL MESSAGES APPEAR IN LOG! ✅ +# =================================================================== +# Log output will show: +# - Driver diagnostics: logger.fine() from connection.py (FINE = 18) +# - Driver diagnostics: logger.finer() from C++ bridge (FINER = 15) ❌ Hidden (15 < 18) +# - User's code: logger.info() messages (INFO = 20) ✅ Visible (20 ≥ 18) +# - User's code: logger.warning() messages (WARNING = 30) ✅ Visible (30 ≥ 18) +# - User's code: logger.error() messages (ERROR = 40) ✅ Visible (40 ≥ 18) +``` + +**Expected Log Output:** +``` +2025-11-03 10:15:22,100 - INFO - app.py:12 - [Python] Attempting database connection... +2025-11-03 10:15:22,101 - FINE - connection.py:42 - [Python] Initializing connection +2025-11-03 10:15:22,102 - FINE - connection.py:56 - [Python] Connection string: Server=localhost;Database=test;... +2025-11-03 10:15:22,110 - FINE - connection.py:89 - [Python] Connection established +2025-11-03 10:15:22,111 - INFO - app.py:16 - [Python] Successfully connected to database +2025-11-03 10:15:22,120 - WARNING - app.py:24 - [Python] Query is very long, may impact performance +2025-11-03 10:15:22,121 - FINE - cursor.py:28 - [Python] Creating cursor +2025-11-03 10:15:22,122 - FINE - cursor.py:89 - [Python] Executing query: SELECT * FROM users +``` + +**Key Takeaway:** Setting `logger.setLevel(FINE)` enables driver diagnostics WITHOUT breaking existing application code that uses `logger.info()`, `logger.warning()`, `logger.error()`! 🎯 + +--- + +### Example 5: Python Code Using Logger (Driver Levels - Recommended) + +```python +""" +connection.py - Example of using Driver Levels logger (recommended for driver code) +""" +from .logging import logger, FINE, FINER, FINEST +from . import ddbc_bindings + +class Connection: + def __init__(self, connection_string: str): + # Use Driver Levels in driver code + logger.fine("Initializing connection") + + # Log sanitized connection string + sanitized = self._sanitize_connection_string(connection_string) + logger.fine(f"Connection string: {sanitized}") + + # Expensive diagnostic only if FINEST enabled + if logger.isEnabledFor(FINEST): + env_info = self._get_environment_info() + logger.finest(f"Environment: {env_info}") + + # Connect via DDBC + self._handle = ddbc_bindings.connect(connection_string) + logger.finer(f"Connection handle allocated: {self._handle}") + + # Generate trace ID + self._trace_id = logger.generate_trace_id("Connection") + logger.fine(f"Connection established [TraceID: {self._trace_id}]") + + def execute(self, sql: str): + logger.fine(f"Executing query: {sql[:100]}...") # Truncate long queries + + if logger.isEnabledFor(FINER): + logger.finer(f"Full query: {sql}") + + result = ddbc_bindings.execute(self._handle, sql) + + logger.finer(f"Query executed, rows affected: {result.rowcount}") + return result + + def close(self): + logger.fine(f"Closing connection [TraceID: {self._trace_id}]") + ddbc_bindings.close(self._handle) + logger.finer("Connection closed successfully") +``` + +--- + +### Example 6: C++ Code Using Logger Bridge + +```cpp +/** + * ddbc_connection.cpp - Example of using logger bridge in C++ + */ +#include "logger_bridge.hpp" +#include + +namespace ddbc { + +class Connection { +public: + Connection(const char* connection_string) { + LOG_FINE("Allocating connection handle"); + + // Allocate ODBC handle + SQLRETURN ret = SQLAllocHandle(SQL_HANDLE_DBC, env_handle_, &handle_); + if (SQL_SUCCEEDED(ret)) { + LOG_FINER("Connection handle allocated: %p", handle_); + } else { + LOG_ERROR("Failed to allocate connection handle, error: %d", ret); + throw ConnectionException("Handle allocation failed"); + } + + // Expensive diagnostic only if FINEST enabled + auto& logger = mssql_python::logging::LoggerBridge; + if (logger::isLoggable(5)) { // FINEST level + std::string diagnostics = getDiagnosticInfo(); + LOG_FINEST("Connection diagnostics: %s", diagnostics.c_str()); + } + + // Connect + LOG_FINE("Connecting to server"); + ret = SQLDriverConnect(handle_, NULL, + (SQLCHAR*)connection_string, SQL_NTS, + NULL, 0, NULL, SQL_DRIVER_NOPROMPT); + + if (SQL_SUCCEEDED(ret)) { + LOG_FINE("Connection established successfully"); + } else { + LOG_ERROR("Connection failed, error: %d", ret); + throw ConnectionException("Connection failed"); + } + } + + void execute(const char* sql) { + LOG_FINE("Executing query: %.100s%s", sql, + strlen(sql) > 100 ? "..." : ""); + + // Full query at FINER level + if (mssql_python::logging::LoggerBridge::isLoggable(15)) { + LOG_FINER("Full query: %s", sql); + } + + SQLRETURN ret = SQLExecDirect(stmt_handle_, (SQLCHAR*)sql, SQL_NTS); + + if (SQL_SUCCEEDED(ret)) { + SQLLEN rowcount; + SQLRowCount(stmt_handle_, &rowcount); + LOG_FINER("Query executed, rows affected: %ld", rowcount); + } else { + LOG_ERROR("Query execution failed, error: %d", ret); + } + } + + ~Connection() { + LOG_FINE("Closing connection handle: %p", handle_); + + if (handle_) { + SQLDisconnect(handle_); + SQLFreeHandle(SQL_HANDLE_DBC, handle_); + LOG_FINER("Connection handle freed"); + } + } + +private: + SQLHDBC handle_; + SQLHSTMT stmt_handle_; + + std::string getDiagnosticInfo() { + // Expensive operation - gather system info + // Only called if FINEST logging enabled + return "...detailed diagnostics..."; + } +}; + +} // namespace ddbc +``` + +--- + +### Example 7: Advanced - Trace ID Usage + +```python +""" +Example: Using Trace IDs to correlate operations +""" +from mssql_python.logging import logger, FINE + +# Enable logging +logger.setLevel(FINE) + +# Create connection (gets trace ID automatically) +conn = mssql_python.connect(connection_string) +print(f"Connection Trace ID: {conn.trace_id}") # e.g., "12345_67890_1" + +# Create cursors (each gets own trace ID) +cursor1 = conn.cursor() +cursor2 = conn.cursor() + +print(f"Cursor 1 Trace ID: {cursor1.trace_id}") # e.g., "12345_67890_2" +print(f"Cursor 2 Trace ID: {cursor2.trace_id}") # e.g., "12345_67890_3" + +# Execute queries - trace IDs appear in logs +cursor1.execute("SELECT * FROM users") +cursor2.execute("SELECT * FROM orders") + +# In logs, you can correlate operations: +# [TraceID: 12345_67890_2] Executing query: SELECT * FROM users +# [TraceID: 12345_67890_3] Executing query: SELECT * FROM orders +``` + +**Log Output**: +``` +2025-10-31 14:30:22,100 - FINE - [TraceID: 12345_67890_1] Connection established +2025-10-31 14:30:22,150 - FINE - [TraceID: 12345_67890_2] Cursor created +2025-10-31 14:30:22,155 - FINE - [TraceID: 12345_67890_2] Executing query: SELECT * FROM users +2025-10-31 14:30:22,160 - FINE - [TraceID: 12345_67890_3] Cursor created +2025-10-31 14:30:22,165 - FINE - [TraceID: 12345_67890_3] Executing query: SELECT * FROM orders +``` + +--- + +## Migration Guide + +### For Users (Application Developers) + +#### Old API (Deprecated) +```python +import mssql_python + +# Old way +mssql_python.setup_logging('stdout') # ❌ Deprecated +``` + +#### New API +```python +from mssql_python.logging import logger, FINE, FINER, FINEST + +# New way - more control +logger.setLevel(FINE) # Standard diagnostics +logger.setLevel(FINER) # Detailed diagnostics +logger.setLevel(FINEST) # Ultra-detailed tracing +logger.setLevel(logging.CRITICAL) # Disable logging +``` + +#### Migration Steps +1. Replace `setup_logging()` calls with `logger.setLevel()` +2. Import logger from `mssql_python.logging` +3. Choose appropriate level (FINE = old default behavior) + +#### Backward Compatibility +```python +# For compatibility, old API still works (deprecated) +def setup_logging(mode='file'): + """Deprecated: Use logger.setLevel() instead""" + from .logging import logger, FINE + logger.setLevel(FINE) + # mode parameter ignored (always logs to file now) +``` + +--- + +### For Contributors (Internal Development) + +#### Python Code Migration + +**Before**: +```python +from .logging_config import LoggingManager + +manager = LoggingManager() +if manager.enabled: + manager.logger.info("[Python Layer log] Connecting...") +``` + +**After**: +```python +from .logging import logger + +logger.fine("Connecting...") # Prefix added automatically +``` + +#### C++ Code Migration + +**Before**: +```cpp +// Old: Always call Python +log_to_python(INFO, "Connecting..."); +``` + +**After**: +```cpp +#include "logger_bridge.hpp" + +// New: Fast check, only call if enabled +LOG_FINE("Connecting..."); + +// For expensive operations +if (LoggerBridge::isLoggable(FINEST)) { + auto details = expensive_operation(); + LOG_FINEST("Details: %s", details.c_str()); +} +``` + +--- + +## Testing Strategy + +### Unit Tests + +#### Python Logger Tests (`test_logging.py`) + +```python +import unittest +import logging +from mssql_python.logging import logger, FINE, FINER, FINEST +import os + +class TestMSSQLLogger(unittest.TestCase): + + def test_custom_levels_defined(self): + """Test that custom levels are registered""" + self.assertEqual(FINE, 25) + self.assertEqual(FINER, 15) + self.assertEqual(FINEST, 5) + self.assertEqual(logging.getLevelName(FINE), 'FINE') + + def test_logger_singleton(self): + """Test that logger is a singleton""" + from mssql_python.logging import MSSQLLogger + logger1 = MSSQLLogger() + logger2 = MSSQLLogger() + self.assertIs(logger1, logger2) + + def test_log_file_created(self): + """Test that log file is created""" + logger.setLevel(FINE) + logger.fine("Test message") + self.assertTrue(os.path.exists(logger.log_file)) + + def test_sanitization(self): + """Test password sanitization""" + logger.setLevel(FINE) + logger.fine("Connection: Server=localhost;PWD=secret123;") + + # Read log file and verify password is sanitized + with open(logger.log_file, 'r') as f: + content = f.read() + self.assertIn("PWD=***", content) + self.assertNotIn("secret123", content) + + def test_level_filtering(self): + """Test that messages are filtered by level""" + logger.setLevel(FINE) + + # FINE should be logged + self.assertTrue(logger.isEnabledFor(FINE)) + + # FINER should not be logged (higher detail) + self.assertFalse(logger.isEnabledFor(FINER)) + + def test_trace_id_generation(self): + """Test trace ID format""" + trace_id = logger.generate_trace_id("Connection") + parts = trace_id.split('_') + + self.assertEqual(len(parts), 3) # PID_ThreadID_Counter + self.assertTrue(all(p.isdigit() for p in parts)) +``` + +#### C++ Bridge Tests (`test_logger_bridge.cpp`) + +```cpp +#include +#include "logger_bridge.hpp" + +using namespace mssql_python::logging; + +class LoggerBridgeTest : public ::testing::Test { +protected: + void SetUp() override { + // Initialize Python interpreter + Py_Initialize(); + LoggerBridge::initialize(); + } + + void TearDown() override { + Py_Finalize(); + } +}; + +TEST_F(LoggerBridgeTest, DefaultLevelIsCritical) { + // By default, logging should be disabled + EXPECT_FALSE(LoggerBridge::isLoggable(25)); // FINE + EXPECT_FALSE(LoggerBridge::isLoggable(15)); // FINER +} + +TEST_F(LoggerBridgeTest, UpdateLevelWorks) { + LoggerBridge::updateLevel(25); // Set to FINE + + EXPECT_TRUE(LoggerBridge::isLoggable(25)); // FINE enabled + EXPECT_FALSE(LoggerBridge::isLoggable(15)); // FINER not enabled +} + +TEST_F(LoggerBridgeTest, LoggingWhenDisabled) { + // Should not crash or call Python + LoggerBridge::updateLevel(50); // CRITICAL (effectively off) + + // This should return immediately + LOG_FINE("This should not be logged"); + LOG_FINER("This should not be logged"); +} + +TEST_F(LoggerBridgeTest, ThreadSafety) { + LoggerBridge::updateLevel(25); + + // Launch multiple threads logging simultaneously + std::vector threads; + for (int i = 0; i < 10; ++i) { + threads.emplace_back([i]() { + for (int j = 0; j < 100; ++j) { + LOG_FINE("Thread %d, message %d", i, j); + } + }); + } + + for (auto& t : threads) { + t.join(); + } + + // Should not crash or corrupt data + SUCCEED(); +} +``` + +--- + +### Integration Tests + +```python +import unittest +import mssql_python +from mssql_python.logging import logger, FINE, FINEST +import os + +class TestLoggingIntegration(unittest.TestCase): + + def setUp(self): + self.connection_string = os.getenv('TEST_CONNECTION_STRING') + logger.setLevel(FINE) + + def test_full_workflow_logged(self): + """Test that complete workflow is logged""" + # Connect + conn = mssql_python.connect(self.connection_string) + + # Execute query + cursor = conn.cursor() + cursor.execute("SELECT 1 as test") + rows = cursor.fetchall() + + # Close + conn.close() + + # Verify log contains expected messages + with open(logger.log_file, 'r') as f: + content = f.read() + + self.assertIn("Initializing connection", content) + self.assertIn("Connection established", content) + self.assertIn("Executing query", content) + self.assertIn("Closing connection", content) + + # Verify C++ logs present + self.assertIn("[DDBC]", content) + + def test_trace_ids_in_logs(self): + """Test that trace IDs appear in logs""" + conn = mssql_python.connect(self.connection_string) + trace_id = conn.trace_id + + cursor = conn.cursor() + cursor.execute("SELECT 1") + + conn.close() + + # Verify trace ID appears in logs + with open(logger.log_file, 'r') as f: + content = f.read() + self.assertIn(f"TraceID: {trace_id}", content) +``` + +--- + +### Performance Tests + +```python +import unittest +import time +import mssql_python +from mssql_python.logging import logger +import logging + +class TestLoggingPerformance(unittest.TestCase): + + def test_overhead_when_disabled(self): + """Test that logging has minimal overhead when disabled""" + logger.setLevel(logging.CRITICAL) # Disable + + conn = mssql_python.connect(self.connection_string) + cursor = conn.cursor() + + # Measure performance with logging disabled + start = time.perf_counter() + for i in range(1000): + cursor.execute("SELECT 1") + disabled_time = time.perf_counter() - start + + # Enable logging + logger.setLevel(FINE) + + # Measure performance with logging enabled + start = time.perf_counter() + for i in range(1000): + cursor.execute("SELECT 1") + enabled_time = time.perf_counter() - start + + # Overhead should be < 10% + overhead = (enabled_time - disabled_time) / disabled_time + self.assertLess(overhead, 0.10, + f"Logging overhead too high: {overhead:.1%}") + + conn.close() +``` + +--- + +## Appendix + +### A. Log Level Decision Tree + +``` +Should I log this message? +│ +├─ Is it always relevant (errors, warnings)? +│ └─ Yes → Use ERROR/WARNING +│ +├─ Is it useful for standard troubleshooting? +│ └─ Yes → Use FINE +│ Examples: +│ - Connection opened/closed +│ - Query executed +│ - Major operations +│ +├─ Is it detailed diagnostic info? +│ └─ Yes → Use FINER +│ Examples: +│ - Handle allocations +│ - Parameter binding +│ - Row counts +│ - Internal state changes +│ +└─ Is it ultra-detailed trace info? + └─ Yes → Use FINEST + Examples: + - Memory dumps + - Full diagnostics + - Performance metrics + - Deep internal state +``` + +### B. C++ Macro Reference + +```cpp +// Driver levels logging macros (used in C++ driver code) +LOG_FINE(fmt, ...) // Standard diagnostics (level 18) +LOG_FINER(fmt, ...) // Detailed diagnostics (level 15) +LOG_FINEST(fmt, ...) // Ultra-detailed trace (level 5) + +// Note: Python standard levels (DEBUG/INFO/WARNING/ERROR) are Python-only. + +// Manual level check for expensive operations +if (LoggerBridge::isLoggable(FINEST)) { + // Expensive computation here +} + +// Example usage patterns +LOG_FINE("Connecting to server: %s", server_name); +LOG_FINER("Handle allocated: %p", handle); +LOG_FINEST("Memory state: %s", dump_memory().c_str()); +``` + +### C. Python API Reference + +```python +from mssql_python.logging import logger, FINE, FINER, FINEST +import logging + +# Driver levels Logging Methods (Recommended for Driver Code) +# ========================================================= +logger.fine(msg) # Standard diagnostics (level 18) +logger.finer(msg) # Detailed diagnostics (level 15) +logger.finest(msg) # Ultra-detailed trace (level 5) + +# Python Standard Logging Methods (Also Available) +# ================================================= +logger.debug(msg) # Debug messages (level 10) +logger.info(msg) # Informational (level 20) +logger.warning(msg) # Warnings (level 30) +logger.error(msg) # Errors (level 40) +logger.critical(msg) # Critical failures (level 50) + +# Level Control +# ====================================== +logger.setLevel(FINE) # Enable FINE and above (includes INFO/WARNING/ERROR) +logger.setLevel(FINER) # Enable FINER and above (includes DEBUG/FINE/INFO/...) +logger.setLevel(FINEST) # Enable everything (most verbose) +logger.setLevel(CRITICAL) # Only critical errors (production default) + +# Level Control (Python standard also works) +# ========================================== +logger.setLevel(logging.DEBUG) # Enable DEBUG and above +logger.setLevel(logging.INFO) # Enable INFO and above +logger.setLevel(logging.WARNING) # Enable WARNING and above + +# Level Checking (for expensive operations) +# ========================================= +if logger.isEnabledFor(FINEST): + expensive_data = compute() + logger.finest(f"Data: {expensive_data}") + +if logger.isEnabledFor(logging.DEBUG): + debug_info = analyze() + logger.debug(f"Info: {debug_info}") + +# Properties +# ========== +logger.log_file # Get current log file path +logger.generate_trace_id(name) # Generate trace ID +``` + +### D. File Structure Summary + +``` +mssql_python/ +├── __init__.py # Export logger +├── logging.py # ← NEW: Main logger (replaces logging_config.py) +├── logging_config.py # ← DEPRECATED: Remove after migration +├── connection.py # Updated: Use new logger +├── cursor.py # Updated: Use new logger +├── auth.py # Updated: Use new logger +├── pooling.py # Updated: Use new logger +│ +└── pybind/ + ├── logger_bridge.hpp # ← NEW: C++ bridge header + ├── logger_bridge.cpp # ← NEW: C++ bridge implementation + ├── bindings.cpp # Updated: Expose bridge functions + ├── ddbc_connection.cpp # Updated: Use LOG_* macros + ├── ddbc_statement.cpp # Updated: Use LOG_* macros + └── ddbc_*.cpp # Updated: Use LOG_* macros + +tests/ +├── test_logging.py # ← NEW: Python logger tests +├── test_logger_bridge.cpp # ← NEW: C++ bridge tests +└── test_logging_integration.py # ← NEW: End-to-end tests + +``` + +### E. Common Troubleshooting + +**Problem**: No logs appearing +**Solution**: Check that `logger.setLevel()` was called with appropriate level + +**Problem**: Passwords appearing in logs +**Solution**: Should never happen - sanitization is automatic. Report as bug. + +**Problem**: Performance degradation +**Solution**: Verify logging is disabled in production, or reduce level to FINE only + +**Problem**: Log file not found +**Solution**: Check `logger.log_file` property for actual location (current working directory) + +**Problem**: C++ logs missing +**Solution**: Verify `LoggerBridge::initialize()` was called during module import + +--- + +## Future Enhancements (Backlog) + +The following items are not part of the initial implementation but are valuable additions for future releases: + +### 1. Cursor.messages Attribute (Priority: High) + +**Inspired By**: PyODBC's `cursor.messages` attribute + +**Description**: Add a `messages` attribute to the Cursor class that captures diagnostic information from the ODBC driver, similar to PyODBC's implementation. + +**Benefits**: +- Provides access to non-error diagnostics (warnings, informational messages) +- Allows users to inspect SQL Server messages without exceptions +- Enables capture of multiple diagnostic records per operation +- Standard pattern familiar to PyODBC users + +**Implementation Details**: +```python +class Cursor: + def __init__(self, connection): + self.messages = [] # List of tuples: (sqlstate, error_code, message) + + def execute(self, sql): + self.messages.clear() # Clear previous messages + # Execute query + # Populate messages from SQLGetDiagRec +``` + +**C++ Support**: +```cpp +// In ddbc_statement.cpp +std::vector> getDiagnosticRecords(SQLHSTMT stmt) { + std::vector> records; + SQLSMALLINT rec_number = 1; + + while (true) { + SQLCHAR sqlstate[6]; + SQLINTEGER native_error; + SQLCHAR message[SQL_MAX_MESSAGE_LENGTH]; + SQLSMALLINT message_len; + + SQLRETURN ret = SQLGetDiagRec(SQL_HANDLE_STMT, stmt, rec_number, + sqlstate, &native_error, + message, sizeof(message), &message_len); + + if (ret == SQL_NO_DATA) break; + if (!SQL_SUCCEEDED(ret)) break; + + records.emplace_back( + std::string((char*)sqlstate), + native_error, + std::string((char*)message) + ); + + rec_number++; + } + + return records; +} +``` + +**Usage Example**: +```python +cursor = conn.cursor() +cursor.execute("SELECT * FROM users") + +# Check for warnings/messages +for sqlstate, error_code, message in cursor.messages: + if sqlstate.startswith('01'): # Warning + print(f"Warning: {message}") +``` + +**Estimated Effort**: 2-3 days + +--- + +### 2. Comprehensive Error Handling via SQLGetDiagRec Chaining (Priority: High) + +**Inspired By**: PyODBC's `GetDiagRecs()` pattern and Psycopg's Diagnostic class + +**Description**: When an error occurs, chain calls to `SQLGetDiagRec` to retrieve ALL diagnostic records, not just the first one. Provide structured access to comprehensive error information. + +**Current Limitation**: +- Errors may only capture the first diagnostic record +- Missing additional context that SQL Server provides +- No structured access to specific diagnostic fields + +**Benefits**: +- Complete error context (multiple records per error) +- Structured diagnostic fields (sqlstate, native_error, message, server, procedure, line) +- Better debugging with full error chains +- More informative exceptions + +**Implementation Details**: + +**Python Exception Enhancement**: +```python +class DatabaseError(Exception): + """Enhanced exception with full diagnostic info""" + def __init__(self, message, diagnostics=None): + super().__init__(message) + self.diagnostics = diagnostics or [] + # diagnostics = [ + # { + # 'sqlstate': '42000', + # 'native_error': 102, + # 'message': 'Incorrect syntax near...', + # 'server': 'myserver', + # 'procedure': 'my_proc', + # 'line': 42 + # }, + # ... + # ] + + def __str__(self): + base = super().__str__() + if self.diagnostics: + diag_info = "\n".join([ + f" [{d['sqlstate']}] {d['message']}" + for d in self.diagnostics + ]) + return f"{base}\nDiagnostics:\n{diag_info}" + return base +``` + +**C++ Diagnostic Retrieval**: +```cpp +// In ddbc_exceptions.cpp +struct DiagnosticRecord { + std::string sqlstate; + int native_error; + std::string message; + std::string server_name; + std::string procedure_name; + int line_number; +}; + +std::vector getAllDiagnostics(SQLHANDLE handle, SQLSMALLINT handle_type) { + std::vector records; + SQLSMALLINT rec_number = 1; + + while (true) { + DiagnosticRecord record; + SQLCHAR sqlstate[6]; + SQLINTEGER native_error; + SQLCHAR message[SQL_MAX_MESSAGE_LENGTH]; + SQLSMALLINT message_len; + + SQLRETURN ret = SQLGetDiagRec(handle_type, handle, rec_number, + sqlstate, &native_error, + message, sizeof(message), &message_len); + + if (ret == SQL_NO_DATA) break; + if (!SQL_SUCCEEDED(ret)) break; + + record.sqlstate = (char*)sqlstate; + record.native_error = native_error; + record.message = (char*)message; + + // Get additional fields via SQLGetDiagField + SQLCHAR server[256]; + ret = SQLGetDiagField(handle_type, handle, rec_number, + SQL_DIAG_SERVER_NAME, server, sizeof(server), NULL); + if (SQL_SUCCEEDED(ret)) { + record.server_name = (char*)server; + } + + // Get procedure name, line number, etc. + // ... + + records.push_back(record); + rec_number++; + + LOG_FINEST("Diagnostic record %d: [%s] %s", rec_number, + record.sqlstate.c_str(), record.message.c_str()); + } + + LOG_FINER("Retrieved %zu diagnostic records", records.size()); + return records; +} +``` + +**Exception Raising with Full Diagnostics**: +```cpp +void raiseException(SQLHANDLE handle, SQLSMALLINT handle_type, const char* operation) { + auto diagnostics = getAllDiagnostics(handle, handle_type); + + if (diagnostics.empty()) { + PyErr_SetString(PyExc_RuntimeError, operation); + return; + } + + // Create Python exception with all diagnostic records + PyObject* diag_list = PyList_New(0); + for (const auto& rec : diagnostics) { + PyObject* diag_dict = Py_BuildValue( + "{s:s, s:i, s:s, s:s, s:s, s:i}", + "sqlstate", rec.sqlstate.c_str(), + "native_error", rec.native_error, + "message", rec.message.c_str(), + "server", rec.server_name.c_str(), + "procedure", rec.procedure_name.c_str(), + "line", rec.line_number + ); + PyList_Append(diag_list, diag_dict); + Py_DECREF(diag_dict); + } + + // Raise DatabaseError with diagnostics + PyObject* exc_class = getExceptionClass(diagnostics[0].sqlstate); + PyObject* exc_instance = PyObject_CallFunction(exc_class, "sO", + diagnostics[0].message.c_str(), + diag_list); + PyErr_SetObject(exc_class, exc_instance); + Py_DECREF(diag_list); + Py_DECREF(exc_instance); +} +``` + +**Usage Example**: +```python +try: + cursor.execute("INVALID SQL") +except mssql_python.DatabaseError as e: + print(f"Error: {e}") + print(f"\nFull diagnostics:") + for diag in e.diagnostics: + print(f" SQLSTATE: {diag['sqlstate']}") + print(f" Native Error: {diag['native_error']}") + print(f" Message: {diag['message']}") + if diag.get('procedure'): + print(f" Procedure: {diag['procedure']} (line {diag['line']})") +``` + +**Estimated Effort**: 3-4 days + +--- + +### Priority and Sequencing + +Both items are marked as **High Priority** for the backlog and should be implemented after the core logging system is complete and stable. + +**Suggested Implementation Order**: +1. Phase 1-4 of core logging system (as described earlier) +2. **Backlog Item #2**: Comprehensive error handling (higher impact on reliability) +3. **Backlog Item #1**: Cursor.messages (complementary diagnostic feature) + +**Dependencies**: +- Both items require the logging system to be in place for proper diagnostic logging +- Item #2 (error handling) benefits from FINEST logging to trace diagnostic retrieval +- Item #1 (cursor.messages) can leverage the same C++ functions as Item #2 + +--- + +## Document History + +| Version | Date | Author | Changes | +| --- | --- | --- | --- | +| 1.0 | 2025-10-31 | Gaurav | Initial design document | +| 1.1 | 2025-10-31 | Gaurav | Added backlog items: cursor.messages and comprehensive error handling | diff --git a/README.md b/README.md index 997cb8ad..696a56e1 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,32 @@ The driver offers a suite of Pythonic enhancements that streamline database inte ### Connection Pooling The Microsoft mssql_python driver provides built-in support for connection pooling, which helps improve performance and scalability by reusing active database connections instead of creating a new connection for every request. This feature is enabled by default. For more information, refer [Connection Pooling Wiki](https://github.com/microsoft/mssql-python/wiki/Connection#connection-pooling). + +### Logging + +The driver includes a comprehensive logging system with JDBC-style custom log levels (FINEST, FINER, FINE) for detailed SQL diagnostics. Features include: + +- **Custom Log Levels**: FINEST (most detailed), FINER, FINE for granular SQL operation logging +- **Automatic Sanitization**: Passwords and sensitive data automatically redacted in logs +- **Trace IDs**: Unique identifiers for tracking related operations +- **File Rotation**: Automatic log file rotation to prevent disk space issues +- **Thread-Safe**: Safe for multi-threaded applications +- **Flexible Logging**: Custom log file paths, rotation, and output control + +Quick example: + +```python +from mssql_python import logger, FINE + +# Enable detailed SQL logging +logger.setLevel(FINE) +logger.enable_file_logging(log_dir='./logs') + +# Use the driver - all SQL operations will be logged +conn = mssql_python.connect(connection_string) +``` + +For complete logging documentation, see [LOGGING.md](LOGGING.md). ## Getting Started Examples Connect to SQL Server and execute a simple query: diff --git a/main.py b/main.py index b45b88d7..2f8cf28c 100644 --- a/main.py +++ b/main.py @@ -1,15 +1,12 @@ from mssql_python import connect -from mssql_python import setup_logging +from mssql_python.logging import setup_logging import os -import decimal -setup_logging('stdout') +# Clean one-liner: set level and output mode together +setup_logging(output="both") conn_str = os.getenv("DB_CONNECTION_STRING") conn = connect(conn_str) - -# conn.autocommit = True - cursor = conn.cursor() cursor.execute("SELECT database_id, name from sys.databases;") rows = cursor.fetchall() diff --git a/mssql_python/__init__.py b/mssql_python/__init__.py index cf510ca2..2f95b41c 100644 --- a/mssql_python/__init__.py +++ b/mssql_python/__init__.py @@ -8,6 +8,8 @@ import types from typing import Dict +from mssql_python.logging import logger + # Import settings from helpers to avoid circular imports from .helpers import Settings, get_settings, _settings, _settings_lock @@ -48,8 +50,8 @@ # Cursor Objects from .cursor import Cursor -# Logging Configuration -from .logging_config import setup_logging, get_logger +# Logging Configuration (Simplified single-level DEBUG system) +from .logging import logger, setup_logging, driver_logger # Constants from .constants import ConstantsDDBC, GetInfoConstants diff --git a/mssql_python/auth.py b/mssql_python/auth.py index b2110fc1..d66e31bb 100644 --- a/mssql_python/auth.py +++ b/mssql_python/auth.py @@ -7,6 +7,8 @@ import platform import struct from typing import Tuple, Dict, Optional, List + +from mssql_python.logging import logger from mssql_python.constants import AuthType diff --git a/mssql_python/connection.py b/mssql_python/connection.py index f0663d72..25de0865 100644 --- a/mssql_python/connection.py +++ b/mssql_python/connection.py @@ -22,9 +22,9 @@ add_driver_to_connection_str, sanitize_connection_string, sanitize_user_input, - log, validate_attribute_value, ) +from mssql_python.logging import logger from mssql_python import ddbc_bindings from mssql_python.pooling import PoolingManager from mssql_python.exceptions import ( @@ -229,6 +229,11 @@ def __init__( # Initialize search escape character self._searchescape = None + # Generate and set trace ID for this connection BEFORE establishing connection + # This ensures all connection establishment logs have the trace ID + self._trace_id = logger.generate_trace_id("CONN") + logger.set_trace_id(self._trace_id) + # Auto-enable pooling if user never called if not PoolingManager.is_initialized(): PoolingManager.enable() @@ -273,7 +278,7 @@ def _construct_connection_string( continue conn_str += f"{key}={value};" - log("info", "Final connection string: %s", sanitize_connection_string(conn_str)) + logger.info( "Final connection string: %s", sanitize_connection_string(conn_str)) return conn_str @@ -308,7 +313,7 @@ def timeout(self, value: int) -> None: if value < 0: raise ValueError("Timeout cannot be negative") self._timeout = value - log("info", f"Query timeout set to {value} seconds") + logger.info( f"Query timeout set to {value} seconds") @property def autocommit(self) -> bool: @@ -329,7 +334,7 @@ def autocommit(self, value: bool) -> None: None """ self.setautocommit(value) - log("info", "Autocommit mode set to %s.", value) + logger.info( "Autocommit mode set to %s.", value) def setautocommit(self, value: bool = False) -> None: """ @@ -374,7 +379,10 @@ def setencoding( # For explicitly using SQL_CHAR cnxn.setencoding(encoding='utf-8', ctype=mssql_python.SQL_CHAR) """ + logger.debug( 'setencoding: Configuring encoding=%s, ctype=%s', + str(encoding) if encoding else 'default', str(ctype) if ctype else 'auto') if self._closed: + logger.debug( 'setencoding: Connection is closed') raise InterfaceError( driver_error="Connection is closed", ddbc_error="Connection is closed", @@ -383,11 +391,12 @@ def setencoding( # Set default encoding if not provided if encoding is None: encoding = "utf-16le" + logger.debug( 'setencoding: Using default encoding=utf-16le') # Validate encoding using cached validation for better performance if not _validate_encoding(encoding): # Log the sanitized encoding for security - log( + logger.debug( "warning", "Invalid encoding attempted: %s", sanitize_user_input(str(encoding)), @@ -399,19 +408,22 @@ def setencoding( # Normalize encoding to casefold for more robust Unicode handling encoding = encoding.casefold() + logger.debug( 'setencoding: Encoding normalized to %s', encoding) # Set default ctype based on encoding if not provided if ctype is None: if encoding in UTF16_ENCODINGS: ctype = ConstantsDDBC.SQL_WCHAR.value + logger.debug( 'setencoding: Auto-selected SQL_WCHAR for UTF-16') else: ctype = ConstantsDDBC.SQL_CHAR.value + logger.debug( 'setencoding: Auto-selected SQL_CHAR for non-UTF-16') # Validate ctype valid_ctypes = [ConstantsDDBC.SQL_CHAR.value, ConstantsDDBC.SQL_WCHAR.value] if ctype not in valid_ctypes: # Log the sanitized ctype for security - log( + logger.debug( "warning", "Invalid ctype attempted: %s", sanitize_user_input(str(ctype)), @@ -428,7 +440,7 @@ def setencoding( self._encoding_settings = {"encoding": encoding, "ctype": ctype} # Log with sanitized values for security - log( + logger.debug( "info", "Text encoding set to %s with ctype %s", sanitize_user_input(encoding), @@ -507,7 +519,7 @@ def setdecoding( SQL_WMETADATA, ] if sqltype not in valid_sqltypes: - log( + logger.debug( "warning", "Invalid sqltype attempted: %s", sanitize_user_input(str(sqltype)), @@ -530,7 +542,7 @@ def setdecoding( # Validate encoding using cached validation for better performance if not _validate_encoding(encoding): - log( + logger.debug( "warning", "Invalid encoding attempted: %s", sanitize_user_input(str(encoding)), @@ -553,7 +565,7 @@ def setdecoding( # Validate ctype valid_ctypes = [ConstantsDDBC.SQL_CHAR.value, ConstantsDDBC.SQL_WCHAR.value] if ctype not in valid_ctypes: - log( + logger.debug( "warning", "Invalid ctype attempted: %s", sanitize_user_input(str(ctype)), @@ -576,7 +588,7 @@ def setdecoding( SQL_WMETADATA: "SQL_WMETADATA", }.get(sqltype, str(sqltype)) - log( + logger.debug( "info", "Text decoding set for %s to %s with ctype %s", sqltype_name, @@ -671,7 +683,7 @@ def set_attr( if not is_valid: # Use the already sanitized values for logging - log( + logger.debug( "warning", f"Invalid attribute or value: {sanitized_attr}={sanitized_val}, {error_message}", ) @@ -681,16 +693,16 @@ def set_attr( ) # Log with sanitized values - log("debug", f"Setting connection attribute: {sanitized_attr}={sanitized_val}") + logger.debug( f"Setting connection attribute: {sanitized_attr}={sanitized_val}") try: # Call the underlying C++ method self._conn.set_attr(attribute, value) - log("info", f"Connection attribute {sanitized_attr} set successfully") + logger.info( f"Connection attribute {sanitized_attr} set successfully") except Exception as e: error_msg = f"Failed to set connection attribute {sanitized_attr}: {str(e)}" - log("error", error_msg) + logger.error( error_msg) # Determine appropriate exception type based on error content error_str = str(e).lower() @@ -725,7 +737,7 @@ def searchescape(self) -> str: self._searchescape = escape_char except Exception as e: # Log the exception for debugging, but do not expose sensitive info - log( + logger.debug( "warning", "Failed to retrieve search escape character, using default '\\'. " "Exception: %s", @@ -789,7 +801,7 @@ def add_output_converter(self, sqltype: int, func: Callable[[Any], Any]) -> None # Pass to the underlying connection if native implementation supports it if hasattr(self._conn, "add_output_converter"): self._conn.add_output_converter(sqltype, func) - log("info", f"Added output converter for SQL type {sqltype}") + logger.info( f"Added output converter for SQL type {sqltype}") def get_output_converter( self, sqltype: Union[int, type] @@ -830,7 +842,7 @@ def remove_output_converter(self, sqltype: Union[int, type]) -> None: # Pass to the underlying connection if native implementation supports it if hasattr(self._conn, "remove_output_converter"): self._conn.remove_output_converter(sqltype) - log("info", f"Removed output converter for SQL type {sqltype}") + logger.info( f"Removed output converter for SQL type {sqltype}") def clear_output_converters(self) -> None: """ @@ -846,7 +858,7 @@ def clear_output_converters(self) -> None: # Pass to the underlying connection if native implementation supports it if hasattr(self._conn, "clear_output_converters"): self._conn.clear_output_converters() - log("info", "Cleared all output converters") + logger.info( "Cleared all output converters") def execute(self, sql: str, *args: Any) -> Cursor: """ @@ -998,11 +1010,11 @@ def batch_execute( # This is an INSERT, UPDATE, DELETE or similar that doesn't return rows results.append(cursor.rowcount) - log("debug", f"Executed batch statement {i+1}/{len(statements)}") + logger.debug( f"Executed batch statement {i+1}/{len(statements)}") except Exception as e: # If a statement fails, include statement context in the error - log( + logger.debug( "error", f"Error executing statement {i+1}/{len(statements)}: {e}", ) @@ -1014,12 +1026,12 @@ def batch_execute( try: # Close the cursor regardless of whether it's reused or new cursor.close() - log( + logger.debug( "debug", "Automatically closed cursor after batch execution error", ) except Exception as close_err: - log( + logger.debug( "warning", f"Error closing cursor after execution failure: {close_err}", ) @@ -1029,7 +1041,7 @@ def batch_execute( # Close the cursor if requested and we created a new one if is_new_cursor and auto_close: cursor.close() - log("debug", "Automatically closed cursor after batch execution") + logger.debug( "Automatically closed cursor after batch execution") return results, cursor @@ -1063,7 +1075,7 @@ def getinfo(self, info_type: int) -> Union[str, int, bool, None]: # Check for invalid info_type values if info_type < 0: - log( + logger.debug( "warning", f"Invalid info_type: {info_type}. Must be a positive integer.", ) @@ -1074,7 +1086,7 @@ def getinfo(self, info_type: int) -> Union[str, int, bool, None]: raw_result = self._conn.get_info(info_type) except Exception as e: # pylint: disable=broad-exception-caught # Log the error and return None for invalid info types - log("warning", f"getinfo({info_type}) failed: {e}") + logger.warning( f"getinfo({info_type}) failed: {e}") return None if raw_result is None: @@ -1091,7 +1103,7 @@ def getinfo(self, info_type: int) -> Union[str, int, bool, None]: length = raw_result["length"] # Debug logging to understand the issue better - log( + logger.debug( "debug", f"getinfo: info_type={info_type}, length={length}, data_type={type(data)}", ) @@ -1167,7 +1179,7 @@ def getinfo(self, info_type: int) -> Union[str, int, bool, None]: try: return actual_data.decode("latin1").rstrip("\0") except Exception as e: - log( + logger.debug( "error", "Failed to decode string in getinfo: %s. " "Returning None to avoid silent corruption.", @@ -1302,7 +1314,7 @@ def commit(self) -> None: # Commit the current transaction self._conn.commit() - log("info", "Transaction committed successfully.") + logger.info( "Transaction committed successfully.") def rollback(self) -> None: """ @@ -1325,7 +1337,7 @@ def rollback(self) -> None: # Roll back the current transaction self._conn.rollback() - log("info", "Transaction rolled back successfully.") + logger.info( "Transaction rolled back successfully.") def close(self) -> None: """ @@ -1357,11 +1369,11 @@ def close(self) -> None: except Exception as e: # pylint: disable=broad-exception-caught # Collect errors but continue closing other cursors close_errors.append(f"Error closing cursor: {e}") - log("warning", f"Error closing cursor: {e}") + logger.warning( f"Error closing cursor: {e}") # If there were errors closing cursors, log them but continue if close_errors: - log( + logger.debug( "warning", "Encountered %d errors while closing cursors", len(close_errors), @@ -1379,24 +1391,23 @@ def close(self) -> None: # This is important to ensure no partial transactions remain # For autocommit True, this is not necessary as each statement is # committed immediately - log( - "info", - "Rolling back uncommitted changes before closing connection.", - ) + logger.debug("Rolling back uncommitted changes before closing connection.") self._conn.rollback() # TODO: Check potential race conditions in case of multithreaded scenarios # Close the connection self._conn.close() self._conn = None except Exception as e: - log("error", f"Error closing database connection: {e}") + logger.error( f"Error closing database connection: {e}") # Re-raise the connection close error as it's more critical raise finally: # Always mark as closed, even if there were errors self._closed = True + # Clear the trace ID context when connection closes + logger.clear_trace_id() - log("info", "Connection closed successfully.") + logger.info( "Connection closed successfully.") def _remove_cursor(self, cursor: Cursor) -> None: """ @@ -1429,7 +1440,7 @@ def __enter__(self) -> "Connection": cursor.execute("INSERT INTO table VALUES (?)", [value]) # Transaction will be committed automatically when exiting """ - log("info", "Entering connection context manager.") + logger.info( "Entering connection context manager.") return self def __exit__(self, *args: Any) -> None: @@ -1455,4 +1466,4 @@ def __del__(self) -> None: self.close() except Exception as e: # Dont raise exceptions from __del__ to avoid issues during garbage collection - log("error", f"Error during connection cleanup: {e}") + logger.error( f"Error during connection cleanup: {e}") diff --git a/mssql_python/constants.py b/mssql_python/constants.py index 785d75e6..37c661f3 100644 --- a/mssql_python/constants.py +++ b/mssql_python/constants.py @@ -6,6 +6,8 @@ from enum import Enum +from mssql_python.logging import logger + class ConstantsDDBC(Enum): """ diff --git a/mssql_python/cursor.py b/mssql_python/cursor.py index 446a2dfb..b9adb71e 100644 --- a/mssql_python/cursor.py +++ b/mssql_python/cursor.py @@ -16,7 +16,8 @@ import warnings from typing import List, Union, Any, Optional, Tuple, Sequence, TYPE_CHECKING from mssql_python.constants import ConstantsDDBC as ddbc_sql_const, SQLTypes -from mssql_python.helpers import check_error, log +from mssql_python.helpers import check_error +from mssql_python.logging import logger from mssql_python import ddbc_bindings from mssql_python.exceptions import InterfaceError, NotSupportedError, ProgrammingError from mssql_python.row import Row @@ -133,6 +134,10 @@ def __init__(self, connection: "Connection", timeout: int = 0) -> None: ) self.messages: List[str] = [] # Store diagnostic messages + + # Generate and set trace ID for this cursor + self._trace_id = logger.generate_trace_id("CURS") + logger.set_trace_id(self._trace_id) def _is_unicode_string(self, param: str) -> bool: """ @@ -306,7 +311,9 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg Returns: - A tuple containing the SQL type, C type, column size, and decimal digits. """ + logger.debug('_map_sql_type: Mapping param index=%d, type=%s', i, type(param).__name__) if param is None: + logger.debug('_map_sql_type: NULL parameter - index=%d', i) return ( ddbc_sql_const.SQL_VARCHAR.value, ddbc_sql_const.SQL_C_DEFAULT.value, @@ -316,6 +323,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg ) if isinstance(param, bool): + logger.debug('_map_sql_type: BOOL detected - index=%d', i) return ( ddbc_sql_const.SQL_BIT.value, ddbc_sql_const.SQL_C_BIT.value, @@ -328,8 +336,11 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg # Use min_val/max_val if available value_to_check = max_val if max_val is not None else param min_to_check = min_val if min_val is not None else param + logger.debug('_map_sql_type: INT detected - index=%d, min=%s, max=%s', + i, str(min_to_check)[:50], str(value_to_check)[:50]) if 0 <= min_to_check and value_to_check <= 255: + logger.debug('_map_sql_type: INT -> TINYINT - index=%d', i) return ( ddbc_sql_const.SQL_TINYINT.value, ddbc_sql_const.SQL_C_TINYINT.value, @@ -338,6 +349,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg False, ) if -32768 <= min_to_check and value_to_check <= 32767: + logger.debug('_map_sql_type: INT -> SMALLINT - index=%d', i) return ( ddbc_sql_const.SQL_SMALLINT.value, ddbc_sql_const.SQL_C_SHORT.value, @@ -346,6 +358,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg False, ) if -2147483648 <= min_to_check and value_to_check <= 2147483647: + logger.debug('_map_sql_type: INT -> INTEGER - index=%d', i) return ( ddbc_sql_const.SQL_INTEGER.value, ddbc_sql_const.SQL_C_LONG.value, @@ -353,6 +366,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg 0, False, ) + logger.debug('_map_sql_type: INT -> BIGINT - index=%d', i) return ( ddbc_sql_const.SQL_BIGINT.value, ddbc_sql_const.SQL_C_SBIGINT.value, @@ -362,6 +376,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg ) if isinstance(param, float): + logger.debug('_map_sql_type: FLOAT detected - index=%d', i) return ( ddbc_sql_const.SQL_DOUBLE.value, ddbc_sql_const.SQL_C_DOUBLE.value, @@ -371,6 +386,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg ) if isinstance(param, decimal.Decimal): + logger.debug('_map_sql_type: DECIMAL detected - index=%d', i) # First check precision limit for all decimal values decimal_as_tuple = param.as_tuple() digits_tuple = decimal_as_tuple.digits @@ -379,6 +395,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg # Handle special values (NaN, Infinity, etc.) if isinstance(exponent, str): + logger.debug('_map_sql_type: DECIMAL special value - index=%d, exponent=%s', i, exponent) # For special values like 'n' (NaN), 'N' (sNaN), 'F' (Infinity) # Return default precision and scale precision = 38 # SQL Server default max precision @@ -390,8 +407,10 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg precision = num_digits else: precision = exponent * -1 + logger.debug('_map_sql_type: DECIMAL precision calculated - index=%d, precision=%d', i, precision) if precision > 38: + logger.debug('_map_sql_type: DECIMAL precision too high - index=%d, precision=%d', i, precision) raise ValueError( f"Precision of the numeric value is too high. " f"The maximum precision supported by SQL Server is 38, but got {precision}." @@ -399,6 +418,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg # Detect MONEY / SMALLMONEY range if SMALLMONEY_MIN <= param <= SMALLMONEY_MAX: + logger.debug('_map_sql_type: DECIMAL -> SMALLMONEY - index=%d', i) # smallmoney parameters_list[i] = str(param) return ( @@ -409,6 +429,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg False, ) if MONEY_MIN <= param <= MONEY_MAX: + logger.debug('_map_sql_type: DECIMAL -> MONEY - index=%d', i) # money parameters_list[i] = str(param) return ( @@ -419,7 +440,10 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg False, ) # fallback to generic numeric binding + logger.debug('_map_sql_type: DECIMAL -> NUMERIC - index=%d', i) parameters_list[i] = self._get_numeric_data(param) + logger.debug('_map_sql_type: NUMERIC created - index=%d, precision=%d, scale=%d', + i, parameters_list[i].precision, parameters_list[i].scale) return ( ddbc_sql_const.SQL_NUMERIC.value, ddbc_sql_const.SQL_C_NUMERIC.value, @@ -429,6 +453,7 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg ) if isinstance(param, uuid.UUID): + logger.debug('_map_sql_type: UUID detected - index=%d', i) parameters_list[i] = param.bytes_le return ( ddbc_sql_const.SQL_GUID.value, @@ -439,11 +464,13 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg ) if isinstance(param, str): + logger.debug('_map_sql_type: STR detected - index=%d, length=%d', i, len(param)) if ( param.startswith("POINT") or param.startswith("LINESTRING") or param.startswith("POLYGON") ): + logger.debug('_map_sql_type: STR is geometry type - index=%d', i) return ( ddbc_sql_const.SQL_WVARCHAR.value, ddbc_sql_const.SQL_C_WCHAR.value, @@ -457,7 +484,10 @@ def _map_sql_type( # pylint: disable=too-many-arguments,too-many-positional-arg # Computes UTF-16 code units (handles surrogate pairs) utf16_len = sum(2 if ord(c) > 0xFFFF else 1 for c in param) + logger.debug('_map_sql_type: STR analysis - index=%d, is_unicode=%s, utf16_len=%d', + i, str(is_unicode), utf16_len) if utf16_len > MAX_INLINE_CHAR: # Long strings -> DAE + logger.debug('_map_sql_type: STR exceeds MAX_INLINE_CHAR, using DAE - index=%d', i) if is_unicode: return ( ddbc_sql_const.SQL_WVARCHAR.value, @@ -571,7 +601,7 @@ def _reset_cursor(self) -> None: if self.hstmt: self.hstmt.free() self.hstmt = None - log("debug", "SQLFreeHandle succeeded") + logger.debug( "SQLFreeHandle succeeded") self._clear_rownumber() @@ -603,14 +633,17 @@ def close(self) -> None: try: self.connection._cursors.discard(self) except Exception as e: # pylint: disable=broad-exception-caught - log("warning", "Error removing cursor from connection tracking: %s", e) + logger.warning( "Error removing cursor from connection tracking: %s", e) if self.hstmt: self.hstmt.free() self.hstmt = None - log("debug", "SQLFreeHandle succeeded") + logger.debug( "SQLFreeHandle succeeded") self._clear_rownumber() self.closed = True + + # Clear the trace ID context when cursor closes + logger.clear_trace_id() def _check_closed(self) -> None: """ @@ -877,7 +910,7 @@ def rownumber(self) -> int: database modules. """ # Use mssql_python logging system instead of standard warnings - log("warning", "DB-API extension cursor.rownumber used") + logger.warning( "DB-API extension cursor.rownumber used") # Return None if cursor is closed or no result set is available if self.closed or not self._has_result_set: @@ -1011,9 +1044,15 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state use_prepare: Whether to use SQLPrepareW (default) or SQLExecDirectW. reset_cursor: Whether to reset the cursor before execution. """ + logger.debug('execute: Starting - operation_length=%d, param_count=%d, use_prepare=%s', + len(operation), len(parameters), str(use_prepare)) + + # Log the actual query being executed + logger.debug('Executing query: %s', operation) # Restore original fetch methods if they exist if hasattr(self, "_original_fetchone"): + logger.debug('execute: Restoring original fetch methods') self.fetchone = self._original_fetchone self.fetchmany = self._original_fetchmany self.fetchall = self._original_fetchall @@ -1023,6 +1062,7 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state self._check_closed() # Check if the cursor is closed if reset_cursor: + logger.debug('execute: Resetting cursor state') self._reset_cursor() # Clear any previous messages @@ -1030,6 +1070,7 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # Apply timeout if set (non-zero) if self._timeout > 0: + logger.debug('execute: Setting query timeout=%d seconds', self._timeout) try: timeout_value = int(self._timeout) ret = ddbc_bindings.DDBCSQLSetStmtAttr( @@ -1038,10 +1079,11 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state timeout_value, ) check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) - log("debug", f"Set query timeout to {timeout_value} seconds") + logger.debug("Set query timeout to %d seconds", timeout_value) except Exception as e: # pylint: disable=broad-exception-caught - log("warning", f"Failed to set query timeout: {e}") + logger.warning("Failed to set query timeout: %s", str(e)) + logger.debug('execute: Creating parameter type list') param_info = ddbc_bindings.ParamInfo parameters_type = [] @@ -1077,10 +1119,8 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # Executing a new statement. Reset is_stmt_prepared to false self.is_stmt_prepared = [False] - log("debug", "Executing query: %s", operation) for i, param in enumerate(parameters): - log( - "debug", + logger.debug( """Parameter number: %s, Parameter: %s, Param Python Type: %s, ParamInfo: %s, %s, %s, %s, %s""", i + 1, @@ -1107,7 +1147,7 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state # Check for errors but don't raise exceptions for info/warning messages check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) except Exception as e: # pylint: disable=broad-exception-caught - log("warning", "Execute failed, resetting cursor: %s", e) + logger.warning( "Execute failed, resetting cursor: %s", e) self._reset_cursor() raise @@ -1149,8 +1189,7 @@ def execute( # pylint: disable=too-many-locals,too-many-branches,too-many-state self._uuid_indices.append(i) # Verify we have complete description tuples (7 items per PEP-249) elif desc and len(desc) != 7: - log( - "warning", + logger.warning( f"Column description at index {i} has incorrect tuple length: {len(desc)}", ) self.rowcount = -1 @@ -1189,11 +1228,10 @@ def _prepare_metadata_result_set( # pylint: disable=too-many-statements try: ddbc_bindings.DDBCSQLDescribeCol(self.hstmt, column_metadata) except InterfaceError as e: - log("error", f"Driver interface error during metadata retrieval: {e}") + logger.error( f"Driver interface error during metadata retrieval: {e}") except Exception as e: # pylint: disable=broad-exception-caught # Log the exception with appropriate context - log( - "error", + logger.error( f"Failed to retrieve column metadata: {e}. " f"Using standard ODBC column definitions instead.", ) @@ -1698,9 +1736,13 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s Raises: Error: If the operation fails. """ + logger.debug( 'executemany: Starting - operation_length=%d, batch_count=%d', + len(operation), len(seq_of_parameters)) + self._check_closed() self._reset_cursor() self.messages = [] + logger.debug( 'executemany: Cursor reset complete') if not seq_of_parameters: self.rowcount = 0 @@ -1716,9 +1758,9 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s timeout_value, ) check_error(ddbc_sql_const.SQL_HANDLE_STMT.value, self.hstmt, ret) - log("debug", f"Set query timeout to {self._timeout} seconds") + logger.debug( f"Set query timeout to {self._timeout} seconds") except Exception as e: # pylint: disable=broad-exception-caught - log("warning", f"Failed to set query timeout: {e}") + logger.warning( f"Failed to set query timeout: {e}") # Get sample row for parameter type detection and validation sample_row = ( @@ -1860,8 +1902,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s any_dae = True if any_dae: - log( - "debug", + logger.debug( "DAE parameters detected. Falling back to row-by-row execution with streaming.", ) for row in seq_of_parameters: @@ -1902,8 +1943,7 @@ def executemany( # pylint: disable=too-many-locals,too-many-branches,too-many-s ) # Add debug logging - log( - "debug", + logger.debug( "Executing batch query with %d parameter sets:\n%s", len(seq_of_parameters), "\n".join( @@ -2223,7 +2263,7 @@ def __del__(self): if sys and sys._is_finalizing(): # Suppress logging during interpreter shutdown return - log("debug", "Exception during cursor cleanup in __del__: %s", e) + logger.debug( "Exception during cursor cleanup in __del__: %s", e) def scroll(self, value: int, mode: str = "relative") -> None: # pylint: disable=too-many-branches """ @@ -2432,7 +2472,7 @@ def tables(self, table=None, catalog=None, schema=None, tableType=None): # pyli except Exception as e: # pylint: disable=broad-exception-caught # Log the error and re-raise - log("error", f"Error executing tables query: {e}") + logger.error( f"Error executing tables query: {e}") raise def callproc( diff --git a/mssql_python/db_connection.py b/mssql_python/db_connection.py index 37bf9b62..73c688da 100644 --- a/mssql_python/db_connection.py +++ b/mssql_python/db_connection.py @@ -5,6 +5,8 @@ """ from typing import Any, Dict, Optional, Union + +from mssql_python.logging import logger from mssql_python.connection import Connection diff --git a/mssql_python/ddbc_bindings.py b/mssql_python/ddbc_bindings.py index bd62050a..06b7697f 100644 --- a/mssql_python/ddbc_bindings.py +++ b/mssql_python/ddbc_bindings.py @@ -10,6 +10,8 @@ import sys import platform +from mssql_python.logging import logger + def normalize_architecture(platform_name_param, architecture_param): """ diff --git a/mssql_python/exceptions.py b/mssql_python/exceptions.py index ff2283f4..cc07f27e 100644 --- a/mssql_python/exceptions.py +++ b/mssql_python/exceptions.py @@ -6,9 +6,7 @@ """ from typing import Optional -from mssql_python.logging_config import get_logger - -logger = get_logger() +from mssql_python.logging import logger class Exception(Exception): @@ -526,8 +524,7 @@ def truncate_error_message(error_message: str) -> str: string_third = string_second[string_second.index("]") + 1 :] return string_first + string_third except Exception as e: - if logger: - logger.error("Error while truncating error message: %s", e) + logger.error("Error while truncating error message: %s", e) return error_message @@ -546,8 +543,7 @@ def raise_exception(sqlstate: str, ddbc_error: str) -> None: """ exception_class = sqlstate_to_exception(sqlstate, ddbc_error) if exception_class: - if logger: - logger.error(exception_class) + logger.error(exception_class) raise exception_class raise DatabaseError( driver_error=f"An error occurred with SQLSTATE code: {sqlstate}", diff --git a/mssql_python/helpers.py b/mssql_python/helpers.py index 1be730ee..e0ee184f 100644 --- a/mssql_python/helpers.py +++ b/mssql_python/helpers.py @@ -10,12 +10,10 @@ from typing import Any, Union, Tuple, Optional from mssql_python import ddbc_bindings from mssql_python.exceptions import raise_exception -from mssql_python.logging_config import get_logger +from mssql_python.logging import logger from mssql_python.constants import ConstantsDDBC # normalize_architecture import removed as it's unused -logger = get_logger() - def add_driver_to_connection_str(connection_str: str) -> str: """ @@ -30,6 +28,7 @@ def add_driver_to_connection_str(connection_str: str) -> str: Raises: Exception: If the connection string is invalid. """ + logger.debug('add_driver_to_connection_str: Processing connection string (length=%d)', len(connection_str)) driver_name = "Driver={ODBC Driver 18 for SQL Server}" try: # Strip any leading or trailing whitespace from the connection string @@ -41,8 +40,11 @@ def add_driver_to_connection_str(connection_str: str) -> str: final_connection_attributes = [] # Iterate through the attributes and exclude any existing driver attribute + driver_found = False for attribute in connection_attributes: if attribute.lower().split("=")[0] == "driver": + driver_found = True + logger.debug('add_driver_to_connection_str: Existing driver attribute found, removing') continue final_connection_attributes.append(attribute) @@ -52,8 +54,11 @@ def add_driver_to_connection_str(connection_str: str) -> str: # Insert the driver attribute at the beginning of the connection string final_connection_attributes.insert(0, driver_name) connection_str = ";".join(final_connection_attributes) + logger.debug('add_driver_to_connection_str: Driver added (had_existing=%s, attr_count=%d)', + str(driver_found), len(final_connection_attributes)) except Exception as e: + logger.debug('add_driver_to_connection_str: Failed to process connection string - %s', str(e)) raise ValueError( "Invalid connection string, Please follow the format: " "Server=server_name;Database=database_name;UID=user_name;PWD=password" @@ -75,9 +80,10 @@ def check_error(handle_type: int, handle: Any, ret: int) -> None: RuntimeError: If an error is found. """ if ret < 0: + logger.debug('check_error: Error detected - handle_type=%d, return_code=%d', handle_type, ret) error_info = ddbc_bindings.DDBCSQLCheckError(handle_type, handle, ret) - if logger: - logger.error("Error: %s", error_info.ddbcErrorMsg) + logger.error("Error: %s", error_info.ddbcErrorMsg) + logger.debug('check_error: SQL state=%s', error_info.sqlState) raise_exception(error_info.sqlState, error_info.ddbcErrorMsg) @@ -91,6 +97,7 @@ def add_driver_name_to_app_parameter(connection_string: str) -> str: Returns: str: The modified connection string. """ + logger.debug('add_driver_name_to_app_parameter: Processing connection string') # Split the input string into key-value pairs parameters = connection_string.split(";") @@ -105,6 +112,7 @@ def add_driver_name_to_app_parameter(connection_string: str) -> str: app_found = True key, _ = param.split("=", 1) modified_parameters.append(f"{key}=MSSQL-Python") + logger.debug('add_driver_name_to_app_parameter: Existing APP parameter overwritten') else: # Keep other parameters as is modified_parameters.append(param) @@ -112,6 +120,7 @@ def add_driver_name_to_app_parameter(connection_string: str) -> str: # If APP key is not found, append it if not app_found: modified_parameters.append("APP=MSSQL-Python") + logger.debug('add_driver_name_to_app_parameter: APP parameter added') # Join the parameters back into a connection string return ";".join(modified_parameters) + ";" @@ -125,9 +134,12 @@ def sanitize_connection_string(conn_str: str) -> str: Returns: str: The sanitized connection string. """ + logger.debug('sanitize_connection_string: Sanitizing connection string (length=%d)', len(conn_str)) # Remove sensitive information from the connection string, Pwd section # Replace Pwd=...; or Pwd=... (end of string) with Pwd=***; - return re.sub(r"(Pwd\s*=\s*)[^;]*", r"\1***", conn_str, flags=re.IGNORECASE) + sanitized = re.sub(r"(Pwd\s*=\s*)[^;]*", r"\1***", conn_str, flags=re.IGNORECASE) + logger.debug('sanitize_connection_string: Password fields masked') + return sanitized def sanitize_user_input(user_input: str, max_length: int = 50) -> str: @@ -142,7 +154,10 @@ def sanitize_user_input(user_input: str, max_length: int = 50) -> str: Returns: str: The sanitized string safe for logging. """ + logger.debug('sanitize_user_input: Sanitizing input (type=%s, length=%d)', + type(user_input).__name__, len(user_input) if isinstance(user_input, str) else 0) if not isinstance(user_input, str): + logger.debug('sanitize_user_input: Non-string input detected') return "" # Remove control characters and non-printable characters @@ -150,11 +165,15 @@ def sanitize_user_input(user_input: str, max_length: int = 50) -> str: sanitized = re.sub(r"[^\w\-\.]", "", user_input) # Limit length to prevent log flooding + was_truncated = False if len(sanitized) > max_length: sanitized = sanitized[:max_length] + "..." + was_truncated = True # Return placeholder if nothing remains after sanitization - return sanitized if sanitized else "" + result = sanitized if sanitized else "" + logger.debug('sanitize_user_input: Result length=%d, truncated=%s', len(result), str(was_truncated)) + return result def validate_attribute_value( @@ -179,6 +198,8 @@ def validate_attribute_value( Returns: tuple: (is_valid, error_message, sanitized_attribute, sanitized_value) """ + logger.debug('validate_attribute_value: Validating attribute=%s, value_type=%s, is_connected=%s', + str(attribute), type(value).__name__, str(is_connected)) # Sanitize a value for logging def _sanitize_for_logging(input_val: Any, max_length: int = max_log_length) -> str: @@ -205,6 +226,7 @@ def _sanitize_for_logging(input_val: Any, max_length: int = max_log_length) -> s # Basic attribute validation - must be an integer if not isinstance(attribute, int): + logger.debug('validate_attribute_value: Attribute not an integer - type=%s', type(attribute).__name__) return ( False, f"Attribute must be an integer, got {type(attribute).__name__}", @@ -224,6 +246,7 @@ def _sanitize_for_logging(input_val: Any, max_length: int = max_log_length) -> s # Check if attribute is supported if attribute not in supported_attributes: + logger.debug('validate_attribute_value: Unsupported attribute - attr=%d', attribute) return ( False, f"Unsupported attribute: {attribute}", @@ -239,6 +262,7 @@ def _sanitize_for_logging(input_val: Any, max_length: int = max_log_length) -> s # Check if attribute can be set at the current connection state if is_connected and attribute in before_only_attributes: + log('finer', 'validate_attribute_value: Timing violation - attr=%d cannot be set after connection', attribute) return ( False, ( @@ -292,21 +316,21 @@ def _sanitize_for_logging(input_val: Any, max_length: int = max_log_length) -> s ) # All basic validations passed + logger.debug('validate_attribute_value: Validation passed - attr=%d, value_type=%s', attribute, type(value).__name__) return True, None, sanitized_attr, sanitized_val def log(level: str, message: str, *args) -> None: """ - Universal logging helper that gets a fresh logger instance. + Universal logging helper that gets the logger instance. Args: - level: Log level ('debug', 'info', 'warning', 'error') + level: Log level ('debug', 'info', 'warning', 'error', 'fine', 'finer', 'finest') message: Log message with optional format placeholders *args: Arguments for message formatting """ - current_logger = get_logger() - if current_logger: - getattr(current_logger, level)(message, *args) + if hasattr(logger, level): + getattr(logger, level)(message, *args) # Settings functionality moved here to avoid circular imports diff --git a/mssql_python/logging.py b/mssql_python/logging.py new file mode 100644 index 00000000..6be488da --- /dev/null +++ b/mssql_python/logging.py @@ -0,0 +1,616 @@ +""" +Copyright (c) Microsoft Corporation. +Licensed under the MIT license. + +Enhanced logging module for mssql_python with JDBC-style logging levels. +This module provides fine-grained logging control with zero overhead when disabled. +""" + +import logging +from logging.handlers import RotatingFileHandler +import os +import sys +import threading +import datetime +import re +import contextvars +import platform +from typing import Optional + + +# Single DEBUG level - all or nothing philosophy +# If you need logging, you need to see everything +DEBUG = logging.DEBUG # 10 + +# Output destination constants +STDOUT = 'stdout' # Log to stdout only +FILE = 'file' # Log to file only (default) +BOTH = 'both' # Log to both file and stdout + +# Allowed log file extensions +ALLOWED_LOG_EXTENSIONS = {'.txt', '.log', '.csv'} + +# Module-level context variable for trace IDs (thread-safe, async-safe) +_trace_id_var = contextvars.ContextVar('trace_id', default=None) + + +class TraceIDFilter(logging.Filter): + """Filter that adds thread_id to all log records.""" + + def filter(self, record): + """Add thread_id (OS native) attribute to log record.""" + # Use OS native thread ID for debugging compatibility + try: + thread_id = threading.get_native_id() + except AttributeError: + # Fallback for Python < 3.8 + thread_id = threading.current_thread().ident + record.thread_id = thread_id + return True + + + + +class MSSQLLogger: + """ + Singleton logger for mssql_python with single DEBUG level. + + Philosophy: All or nothing - if you enable logging, you see EVERYTHING. + Logging is a troubleshooting tool, not a production feature. + + Features: + - Single DEBUG level (no categorization) + - Automatic file rotation (512MB, 5 backups) + - Password sanitization + - Trace ID support with contextvars (automatic propagation) + - Thread-safe operation + - Zero overhead when disabled (level check only) + + ⚠️ Performance Warning: Logging adds ~2-5% overhead. Only enable when troubleshooting. + """ + + _instance: Optional['MSSQLLogger'] = None + _lock = threading.Lock() + + def __new__(cls) -> 'MSSQLLogger': + """Ensure singleton pattern""" + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super(MSSQLLogger, cls).__new__(cls) + return cls._instance + + def __init__(self): + """Initialize the logger (only once)""" + # Skip if already initialized + if hasattr(self, '_initialized'): + return + + self._initialized = True + + # Create the underlying Python logger + self._logger = logging.getLogger('mssql_python') + self._logger.setLevel(logging.CRITICAL) # Disabled by default + self._logger.propagate = False # Don't propagate to root logger + + # Add trace ID filter (injects trace_id into every log record) + self._logger.addFilter(TraceIDFilter()) + + # Trace ID counter (thread-safe) + self._trace_counter = 0 + self._trace_lock = threading.Lock() + + # Output mode and handlers + self._output_mode = FILE # Default to file only + self._file_handler = None + self._stdout_handler = None + self._log_file = None + self._custom_log_path = None # Custom log file path (if specified) + self._handlers_initialized = False + + # Don't setup handlers yet - do it lazily when setLevel is called + # This prevents creating log files when user changes output mode before enabling logging + + def _setup_handlers(self): + """ + Setup handlers based on output mode. + Creates file handler and/or stdout handler as needed. + """ + # Clear any existing handlers + if self._logger.handlers: + for handler in self._logger.handlers[:]: + handler.close() + self._logger.removeHandler(handler) + + self._file_handler = None + self._stdout_handler = None + + # Create CSV formatter + # Custom formatter to extract source from message and format as CSV + class CSVFormatter(logging.Formatter): + def format(self, record): + # Extract source from message (e.g., [Python] or [DDBC]) + msg = record.getMessage() + if msg.startswith('[') and ']' in msg: + end_bracket = msg.index(']') + source = msg[1:end_bracket] + message = msg[end_bracket+2:].strip() # Skip '] ' + else: + source = 'Unknown' + message = msg + + # Format timestamp with milliseconds using period separator + timestamp = self.formatTime(record, '%Y-%m-%d %H:%M:%S') + timestamp_with_ms = f"{timestamp}.{int(record.msecs):03d}" + + # Get thread ID + thread_id = getattr(record, 'thread_id', 0) + + # Build CSV row + location = f"{record.filename}:{record.lineno}" + csv_row = f"{timestamp_with_ms}, {thread_id}, {record.levelname}, {location}, {source}, {message}" + + return csv_row + + formatter = CSVFormatter() + + # Override format to use milliseconds with period separator + formatter.default_msec_format = '%s.%03d' + + # Setup file handler if needed + if self._output_mode in (FILE, BOTH): + # Use custom path or auto-generate + if self._custom_log_path: + self._log_file = self._custom_log_path + # Ensure directory exists for custom path + log_dir = os.path.dirname(self._custom_log_path) + if log_dir and not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + else: + # Create log file in mssql_python_logs folder + log_dir = os.path.join(os.getcwd(), "mssql_python_logs") + if not os.path.exists(log_dir): + os.makedirs(log_dir, exist_ok=True) + + timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + pid = os.getpid() + self._log_file = os.path.join( + log_dir, + f"mssql_python_trace_{timestamp}_{pid}.log" + ) + + # Create rotating file handler (512MB, 5 backups) + self._file_handler = RotatingFileHandler( + self._log_file, + maxBytes=512 * 1024 * 1024, # 512MB + backupCount=5 + ) + self._file_handler.setFormatter(formatter) + self._logger.addHandler(self._file_handler) + + # Write CSV header to new log file + self._write_log_header() + else: + # No file logging - clear the log file path + self._log_file = None + + # Setup stdout handler if needed + if self._output_mode in (STDOUT, BOTH): + import sys + self._stdout_handler = logging.StreamHandler(sys.stdout) + self._stdout_handler.setFormatter(formatter) + self._logger.addHandler(self._stdout_handler) + + def _reconfigure_handlers(self): + """ + Reconfigure handlers when output mode changes. + Closes existing handlers and creates new ones based on current output mode. + """ + self._setup_handlers() + + def _validate_log_file_extension(self, file_path: str) -> None: + """ + Validate that the log file has an allowed extension. + + Args: + file_path: Path to the log file + + Raises: + ValueError: If the file extension is not allowed + """ + _, ext = os.path.splitext(file_path) + ext_lower = ext.lower() + + if ext_lower not in ALLOWED_LOG_EXTENSIONS: + allowed = ', '.join(sorted(ALLOWED_LOG_EXTENSIONS)) + raise ValueError( + f"Invalid log file extension '{ext}'. " + f"Allowed extensions: {allowed}" + ) + + def _write_log_header(self): + """ + Write CSV header and metadata to the log file. + Called once when log file is created. + """ + if not self._log_file or not self._file_handler: + return + + try: + # Get script name from sys.argv or __main__ + script_name = os.path.basename(sys.argv[0]) if sys.argv else '' + + # Get Python version + python_version = platform.python_version() + + # Get driver version (try to import from package) + try: + from mssql_python import __version__ + driver_version = __version__ + except: + driver_version = 'unknown' + + # Get current time + start_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + # Get PID + pid = os.getpid() + + # Get OS info + os_info = platform.platform() + + # Build header comment line + header_line = f"# MSSQL-Python Driver Log | Script: {script_name} | PID: {pid} | Log Level: DEBUG | Python: {python_version} | Driver: {driver_version} | Start: {start_time} | OS: {os_info}\n" + + # CSV column headers + csv_header = "Timestamp, ThreadID, Level, Location, Source, Message\n" + + # Write directly to file (bypass formatter) + with open(self._log_file, 'a') as f: + f.write(header_line) + f.write(csv_header) + + except Exception as e: + # Don't fail if header writing fails + pass + + @staticmethod + def _sanitize_message(msg: str) -> str: + """ + Sanitize sensitive information from log messages. + + Removes: + - PWD=... + - Password=... + - TOKEN=... + - Authorization: Bearer ... + + Args: + msg: The message to sanitize + + Returns: + str: Sanitized message with credentials replaced by *** + """ + # Pattern to match various credential formats + patterns = [ + (r'(PWD|Password|pwd|password)\s*=\s*[^;,\s]+', r'\1=***'), + (r'(TOKEN|Token|token)\s*=\s*[^;,\s]+', r'\1=***'), + (r'(Authorization:\s*Bearer\s+)[^\s;,]+', r'\1***'), + (r'(ApiKey|API_KEY|api_key)\s*=\s*[^;,\s]+', r'\1=***'), + ] + + sanitized = msg + for pattern, replacement in patterns: + sanitized = re.sub(pattern, replacement, sanitized) + + return sanitized + + def generate_trace_id(self, prefix: str = "TRACE") -> str: + """ + Generate a unique trace ID for correlating log messages. + + Format: PREFIX-PID-ThreadID-Counter + Examples: + CONN-12345-67890-1 + CURS-12345-67890-2 + + Args: + prefix: Prefix for the trace ID (e.g., "CONN", "CURS", "TRACE") + + Returns: + str: Unique trace ID in format PREFIX-PID-ThreadID-Counter + """ + with self._trace_lock: + self._trace_counter += 1 + counter = self._trace_counter + + pid = os.getpid() + thread_id = threading.get_ident() + + return f"{prefix}-{pid}-{thread_id}-{counter}" + + def set_trace_id(self, trace_id: str): + """ + Set the trace ID for the current context. + + This uses contextvars, so the trace ID automatically propagates to: + - Child threads created within this context + - Async tasks spawned from this context + - All log calls made within this context + + Args: + trace_id: Trace ID to set (typically from generate_trace_id()) + + Example: + trace_id = logger.generate_trace_id("CONN") + logger.set_trace_id(trace_id) + logger.debug("Connection opened") # Includes trace ID automatically + """ + _trace_id_var.set(trace_id) + + def get_trace_id(self) -> Optional[str]: + """ + Get the trace ID for the current context. + + Returns: + str or None: Current trace ID, or None if not set + """ + return _trace_id_var.get() + + def clear_trace_id(self): + """ + Clear the trace ID for the current context. + + Typically called when closing a connection/cursor to avoid + trace ID leaking to subsequent operations. + """ + _trace_id_var.set(None) + + def _log(self, level: int, msg: str, *args, **kwargs): + """ + Internal logging method with sanitization. + + Args: + level: Log level (FINE, FINER, FINEST, etc.) + msg: Message format string + *args: Arguments for message formatting + **kwargs: Additional keyword arguments + """ + # Fast level check (zero overhead if disabled) + if not self._logger.isEnabledFor(level): + return + + # Format message with args if provided + if args: + msg = msg % args + + # Sanitize message + sanitized_msg = self._sanitize_message(msg) + + # Log the message (no args since already formatted) + self._logger.log(level, sanitized_msg, **kwargs) + + # Convenience methods for logging + + def debug(self, msg: str, *args, **kwargs): + """Log at DEBUG level (all diagnostic messages)""" + self._log(logging.DEBUG, f"[Python] {msg}", *args, **kwargs) + + def info(self, msg: str, *args, **kwargs): + """Log at INFO level""" + self._log(logging.INFO, f"[Python] {msg}", *args, **kwargs) + + def warning(self, msg: str, *args, **kwargs): + """Log at WARNING level""" + self._log(logging.WARNING, f"[Python] {msg}", *args, **kwargs) + + def error(self, msg: str, *args, **kwargs): + """Log at ERROR level""" + self._log(logging.ERROR, f"[Python] {msg}", *args, **kwargs) + + def critical(self, msg: str, *args, **kwargs): + """Log at CRITICAL level""" + self._log(logging.CRITICAL, f"[Python] {msg}", *args, **kwargs) + + def log(self, level: int, msg: str, *args, **kwargs): + """Log a message at the specified level""" + self._log(level, f"[Python] {msg}", *args, **kwargs) + + # Level control + + def _setLevel(self, level: int, output: Optional[str] = None, log_file_path: Optional[str] = None): + """ + Internal method to set logging level (use setup_logging() instead). + + Args: + level: Logging level (typically DEBUG) + output: Optional output mode (FILE, STDOUT, BOTH) + log_file_path: Optional custom path for log file + + Raises: + ValueError: If output mode is invalid + """ + # Validate and set output mode if specified + if output is not None: + if output not in (FILE, STDOUT, BOTH): + raise ValueError( + f"Invalid output mode: {output}. " + f"Must be one of: {FILE}, {STDOUT}, {BOTH}" + ) + self._output_mode = output + + # Store custom log file path if provided + if log_file_path is not None: + self._validate_log_file_extension(log_file_path) + self._custom_log_path = log_file_path + + # Setup handlers if not yet initialized or if output mode/path changed + if not self._handlers_initialized or output is not None or log_file_path is not None: + self._setup_handlers() + self._handlers_initialized = True + + # Set level + self._logger.setLevel(level) + + # Notify C++ bridge of level change + self._notify_cpp_level_change(level) + + def getLevel(self) -> int: + """ + Get the current logging level. + + Returns: + int: Current log level + """ + return self._logger.level + + def isEnabledFor(self, level: int) -> bool: + """ + Check if a given log level is enabled. + + Args: + level: Log level to check + + Returns: + bool: True if the level is enabled + """ + return self._logger.isEnabledFor(level) + + # Handler management + + def addHandler(self, handler: logging.Handler): + """Add a handler to the logger""" + self._logger.addHandler(handler) + + def removeHandler(self, handler: logging.Handler): + """Remove a handler from the logger""" + self._logger.removeHandler(handler) + + @property + def handlers(self) -> list: + """Get list of handlers attached to the logger""" + return self._logger.handlers + + def reset_handlers(self): + """ + Reset/recreate handlers. + Useful when log file has been deleted or needs to be recreated. + """ + self._setup_handlers() + + def _notify_cpp_level_change(self, level: int): + """ + Notify C++ bridge that log level has changed. + This updates the cached level in C++ for fast checks. + + Args: + level: New log level + """ + try: + # Import here to avoid circular dependency + from . import ddbc_bindings + if hasattr(ddbc_bindings, 'update_log_level'): + ddbc_bindings.update_log_level(level) + except (ImportError, AttributeError): + # C++ bindings not available or not yet initialized + pass + + # Properties + + @property + def output(self) -> str: + """Get the current output mode""" + return self._output_mode + + @output.setter + def output(self, mode: str): + """ + Set the output mode. + + Args: + mode: Output mode (FILE, STDOUT, or BOTH) + + Raises: + ValueError: If mode is not a valid OutputMode value + """ + if mode not in (FILE, STDOUT, BOTH): + raise ValueError( + f"Invalid output mode: {mode}. " + f"Must be one of: {FILE}, {STDOUT}, {BOTH}" + ) + self._output_mode = mode + + # Only reconfigure if handlers were already initialized + if self._handlers_initialized: + self._reconfigure_handlers() + + @property + def log_file(self) -> Optional[str]: + """Get the current log file path (None if file output is disabled)""" + return self._log_file + + @property + def level(self) -> int: + """Get the current logging level""" + return self._logger.level + + +# ============================================================================ +# Module-level exports (Primary API) +# ============================================================================ + +# Singleton logger instance +logger = MSSQLLogger() + +# Expose the underlying Python logger for use in application code +# This allows applications to access the same logger used by the driver +# Usage: from mssql_python.logging import driver_logger +driver_logger = logger._logger + +# ============================================================================ +# Primary API - setup_logging() +# ============================================================================ + +def setup_logging(output: str = 'file', log_file_path: Optional[str] = None): + """ + Enable DEBUG logging for troubleshooting. + + ⚠️ PERFORMANCE WARNING: Logging adds ~2-5% overhead. + Only enable when investigating issues. Do NOT enable in production without reason. + + Philosophy: All or nothing - if you need logging, you need to see EVERYTHING. + Logging is a troubleshooting tool, not a production monitoring solution. + + Args: + output: Where to send logs (default: 'file') + Options: 'file', 'stdout', 'both' + log_file_path: Optional custom path for log file + Must have extension: .txt, .log, or .csv + If not specified, auto-generates in ./mssql_python_logs/ + + Examples: + import mssql_python + + # File only (default, in mssql_python_logs folder) + mssql_python.setup_logging() + + # Stdout only (for CI/CD) + mssql_python.setup_logging(output='stdout') + + # Both file and stdout (for development) + mssql_python.setup_logging(output='both') + + # Custom log file path (must use .txt, .log, or .csv extension) + mssql_python.setup_logging(log_file_path="/var/log/myapp.log") + mssql_python.setup_logging(log_file_path="/tmp/debug.txt") + mssql_python.setup_logging(log_file_path="/tmp/data.csv") + + # Custom path with both outputs + mssql_python.setup_logging(output='both', log_file_path="/tmp/debug.log") + + Future Enhancement: + For performance analysis, use the universal profiler (coming soon) + instead of logging. Logging is not designed for performance measurement. + """ + logger._setLevel(logging.DEBUG, output, log_file_path) + return logger diff --git a/mssql_python/logging_config.py b/mssql_python/logging_config.py deleted file mode 100644 index f826092a..00000000 --- a/mssql_python/logging_config.py +++ /dev/null @@ -1,181 +0,0 @@ -""" -Copyright (c) Microsoft Corporation. -Licensed under the MIT license. -This module provides logging configuration for the mssql_python package. -""" - -import logging -from logging.handlers import RotatingFileHandler -import os -import sys -import datetime -from typing import Optional - - -class LoggingManager: - """ - Singleton class to manage logging configuration for the mssql_python package. - This class provides a centralized way to manage logging configuration and replaces - the previous approach using global variables. - """ - - _instance: Optional["LoggingManager"] = None - _initialized: bool = False - _logger: Optional[logging.Logger] = None - _log_file: Optional[str] = None - - def __new__(cls) -> "LoggingManager": - if cls._instance is None: - cls._instance = super(LoggingManager, cls).__new__(cls) - return cls._instance - - def __init__(self) -> None: - if not self._initialized: - self._initialized = True - self._enabled = False - - @classmethod - def is_logging_enabled(cls) -> bool: - """Class method to check if logging is enabled for backward compatibility""" - if cls._instance is None: - return False - return cls._instance._enabled - - @property - def enabled(self) -> bool: - """Check if logging is enabled""" - return self._enabled - - @property - def log_file(self) -> Optional[str]: - """Get the current log file path""" - return self._log_file - - def setup( - self, mode: str = "file", log_level: int = logging.DEBUG - ) -> Optional[logging.Logger]: - """ - Set up logging configuration. - - This method configures the logging settings for the application. - It sets the log level, format, and log file location. - - Args: - mode (str): The logging mode ('file' or 'stdout'). - log_level (int): The logging level (default: logging.DEBUG). - """ - # Enable logging - self._enabled = True - - # Create a logger for mssql_python module - # Use a consistent logger name to ensure we're using the same logger throughout - self._logger = logging.getLogger("mssql_python") - self._logger.setLevel(log_level) - - # Configure the root logger to ensure all messages are captured - root_logger = logging.getLogger() - root_logger.setLevel(log_level) - - # Make sure the logger propagates to the root logger - self._logger.propagate = True - - # Clear any existing handlers to avoid duplicates during re-initialization - if self._logger.handlers: - self._logger.handlers.clear() - - # Construct the path to the log file - # Directory for log files - currentdir/logs - current_dir = os.path.dirname(os.path.abspath(__file__)) - log_dir = os.path.join(current_dir, "logs") - # exist_ok=True allows the directory to be created if it doesn't exist - os.makedirs(log_dir, exist_ok=True) - - # Generate timestamp-based filename for better sorting and organization - timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") - self._log_file = os.path.join( - log_dir, f"mssql_python_trace_{timestamp}_{os.getpid()}.log" - ) - - # Create a log handler to log to driver specific file - # By default we only want to log to a file, max size 500MB, and keep 5 backups - file_handler = RotatingFileHandler( - self._log_file, maxBytes=512 * 1024 * 1024, backupCount=5 - ) - file_handler.setLevel(log_level) - - # Create a custom formatter that adds [Python Layer log] prefix only to non-DDBC messages - class PythonLayerFormatter(logging.Formatter): - """Custom formatter that adds [Python Layer log] prefix to non-DDBC messages.""" - def format(self, record): - message = record.getMessage() - # Don't add [Python Layer log] prefix if the message already has - # [DDBC Bindings log] or [Python Layer log] - if ( - "[DDBC Bindings log]" not in message - and "[Python Layer log]" not in message - ): - # Create a copy of the record to avoid modifying the original - new_record = logging.makeLogRecord(record.__dict__) - new_record.msg = f"[Python Layer log] {record.msg}" - return super().format(new_record) - return super().format(record) - - # Use our custom formatter - formatter = PythonLayerFormatter( - "%(asctime)s - %(levelname)s - %(filename)s - %(message)s" - ) - file_handler.setFormatter(formatter) - self._logger.addHandler(file_handler) - - if mode == "stdout": - # If the mode is stdout, then we want to log to the console as well - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.setLevel(log_level) - # Use the same smart formatter - stdout_handler.setFormatter(formatter) - self._logger.addHandler(stdout_handler) - elif mode != "file": - raise ValueError(f"Invalid logging mode: {mode}") - - return self._logger - - def get_logger(self) -> Optional[logging.Logger]: - """ - Get the logger instance. - - Returns: - logging.Logger: The logger instance, or None if logging is not enabled. - """ - if not self.enabled: - # If logging is not enabled, return None - return None - return self._logger - - -# Create a singleton instance -_manager = LoggingManager() - - -def setup_logging(mode: str = "file", log_level: int = logging.DEBUG) -> None: - """ - Set up logging configuration. - - This is a wrapper around the LoggingManager.setup method for backward compatibility. - - Args: - mode (str): The logging mode ('file' or 'stdout'). - log_level (int): The logging level (default: logging.DEBUG). - """ - return _manager.setup(mode, log_level) - - -def get_logger() -> Optional[logging.Logger]: - """ - Get the logger instance. - - This is a wrapper around the LoggingManager.get_logger method for backward compatibility. - - Returns: - logging.Logger: The logger instance. - """ - return _manager.get_logger() diff --git a/mssql_python/pooling.py b/mssql_python/pooling.py index 88e1b624..3122369c 100644 --- a/mssql_python/pooling.py +++ b/mssql_python/pooling.py @@ -7,6 +7,7 @@ import threading from typing import Dict +from mssql_python.logging import logger from mssql_python import ddbc_bindings diff --git a/mssql_python/pybind/CMakeLists.txt b/mssql_python/pybind/CMakeLists.txt index 489dfd45..358e0bbb 100644 --- a/mssql_python/pybind/CMakeLists.txt +++ b/mssql_python/pybind/CMakeLists.txt @@ -186,8 +186,8 @@ message(STATUS "Final Python library directory: ${PYTHON_LIB_DIR}") set(DDBC_SOURCE "ddbc_bindings.cpp") message(STATUS "Using standard source file: ${DDBC_SOURCE}") -# Include connection module for Windows -add_library(ddbc_bindings MODULE ${DDBC_SOURCE} connection/connection.cpp connection/connection_pool.cpp) +# Include connection module and logger bridge +add_library(ddbc_bindings MODULE ${DDBC_SOURCE} connection/connection.cpp connection/connection_pool.cpp logger_bridge.cpp) # Set the output name to include Python version and architecture # Use appropriate file extension based on platform diff --git a/mssql_python/pybind/connection/connection.cpp b/mssql_python/pybind/connection/connection.cpp index a22f4c8e..517284ad 100644 --- a/mssql_python/pybind/connection/connection.cpp +++ b/mssql_python/pybind/connection/connection.cpp @@ -14,6 +14,9 @@ #define SQL_COPT_SS_ACCESS_TOKEN 1256 // Custom attribute ID for access token #define SQL_MAX_SMALL_INT 32767 // Maximum value for SQLSMALLINT +// Logging uses LOG() macro for all diagnostic output +#include "logger_bridge.hpp" + static SqlHandlePtr getEnvHandle() { static SqlHandlePtr envHandle = []() -> SqlHandlePtr { LOG("Allocating ODBC environment handle"); @@ -57,7 +60,7 @@ Connection::~Connection() { void Connection::allocateDbcHandle() { auto _envHandle = getEnvHandle(); SQLHANDLE dbc = nullptr; - LOG("Allocate SQL Connection Handle"); + LOG("Allocating SQL Connection Handle"); SQLRETURN ret = SQLAllocHandle_ptr(SQL_HANDLE_DBC, _envHandle->get(), &dbc); checkError(ret); @@ -80,7 +83,7 @@ void Connection::connect(const py::dict& attrs_before) { LOG("Creating connection string buffer for macOS/Linux"); std::vector connStrBuffer = WStringToSQLWCHAR(_connStr); // Ensure the buffer is null-terminated - LOG("Connection string buffer size - {}", connStrBuffer.size()); + LOG("Connection string buffer size=%zu", connStrBuffer.size()); connStrPtr = connStrBuffer.data(); LOG("Connection string buffer created"); #else @@ -143,15 +146,15 @@ void Connection::setAutocommit(bool enable) { ThrowStdException("Connection handle not allocated"); } SQLINTEGER value = enable ? SQL_AUTOCOMMIT_ON : SQL_AUTOCOMMIT_OFF; - LOG("Setting SQL Connection Attribute"); + LOG("Setting autocommit=%d", enable); SQLRETURN ret = SQLSetConnectAttr_ptr( _dbcHandle->get(), SQL_ATTR_AUTOCOMMIT, reinterpret_cast(static_cast(value)), 0); checkError(ret); if (value == SQL_AUTOCOMMIT_ON) { - LOG("SQL Autocommit set to True"); + LOG("Autocommit enabled"); } else { - LOG("SQL Autocommit set to False"); + LOG("Autocommit disabled"); } _autocommit = enable; } @@ -160,7 +163,7 @@ bool Connection::getAutocommit() const { if (!_dbcHandle) { ThrowStdException("Connection handle not allocated"); } - LOG("Get SQL Connection Attribute"); + LOG("Getting autocommit attribute"); SQLINTEGER value; SQLINTEGER string_length; SQLRETURN ret = SQLGetConnectAttr_ptr(_dbcHandle->get(), @@ -185,7 +188,7 @@ SqlHandlePtr Connection::allocStatementHandle() { } SQLRETURN Connection::setAttribute(SQLINTEGER attribute, py::object value) { - LOG("Setting SQL attribute"); + LOG("Setting SQL attribute=%d", attribute); // SQLPOINTER ptr = nullptr; // SQLINTEGER length = 0; static std::string buffer; // to hold sensitive data temporarily @@ -201,9 +204,9 @@ SQLRETURN Connection::setAttribute(SQLINTEGER attribute, py::object value) { SQL_IS_INTEGER); if (!SQL_SUCCEEDED(ret)) { - LOG("Failed to set attribute"); + LOG("Failed to set integer attribute=%d, ret=%d", attribute, ret); } else { - LOG("Set attribute successfully"); + LOG("Set integer attribute=%d successfully", attribute); } return ret; } else if (py::isinstance(value)) { @@ -215,7 +218,7 @@ SQLRETURN Connection::setAttribute(SQLINTEGER attribute, py::object value) { // Convert to wide string std::wstring wstr = Utf8ToWString(utf8_str); if (wstr.empty() && !utf8_str.empty()) { - LOG("Failed to convert string value to wide string"); + LOG("Failed to convert string value to wide string for attribute=%d", attribute); return SQL_ERROR; } @@ -236,7 +239,7 @@ SQLRETURN Connection::setAttribute(SQLINTEGER attribute, py::object value) { // For macOS/Linux, convert wstring to SQLWCHAR buffer std::vector sqlwcharBuffer = WStringToSQLWCHAR(wstr); if (sqlwcharBuffer.empty() && !wstr.empty()) { - LOG("Failed to convert wide string to SQLWCHAR buffer"); + LOG("Failed to convert wide string to SQLWCHAR buffer for attribute=%d", attribute); return SQL_ERROR; } @@ -253,14 +256,13 @@ SQLRETURN Connection::setAttribute(SQLINTEGER attribute, py::object value) { SQLRETURN ret = SQLSetConnectAttr_ptr(_dbcHandle->get(), attribute, ptr, length); if (!SQL_SUCCEEDED(ret)) { - LOG("Failed to set string attribute"); + LOG("Failed to set string attribute=%d, ret=%d", attribute, ret); } else { - LOG("Set string attribute successfully"); + LOG("Set string attribute=%d successfully", attribute); } return ret; } catch (const std::exception& e) { - LOG("Exception during string attribute setting: " + - std::string(e.what())); + LOG("Exception during string attribute=%d setting: %s", attribute, e.what()); return SQL_ERROR; } } else if (py::isinstance(value) || @@ -285,18 +287,17 @@ SQLRETURN Connection::setAttribute(SQLINTEGER attribute, py::object value) { SQLRETURN ret = SQLSetConnectAttr_ptr(_dbcHandle->get(), attribute, ptr, length); if (!SQL_SUCCEEDED(ret)) { - LOG("Failed to set attribute with binary data"); + LOG("Failed to set binary attribute=%d, ret=%d", attribute, ret); } else { - LOG("Set attribute successfully with binary data"); + LOG("Set binary attribute=%d successfully (length=%d)", attribute, length); } return ret; } catch (const std::exception& e) { - LOG("Exception during binary attribute setting: " + - std::string(e.what())); + LOG("Exception during binary attribute=%d setting: %s", attribute, e.what()); return SQL_ERROR; } } else { - LOG("Unsupported attribute value type"); + LOG("Unsupported attribute value type for attribute=%d", attribute); return SQL_ERROR; } } @@ -344,7 +345,7 @@ bool Connection::reset() { (SQLPOINTER)SQL_RESET_CONNECTION_YES, SQL_IS_INTEGER); if (!SQL_SUCCEEDED(ret)) { - LOG("Failed to reset connection. Marking as dead."); + LOG("Failed to reset connection (ret=%d). Marking as dead.", ret); disconnect(); return false; } @@ -516,13 +517,13 @@ void ConnectionHandle::setAttr(int attribute, py::object value) { errorMsg += ": " + ddbcErrorStr; } - LOG("Connection setAttribute failed: {}", errorMsg); + LOG("Connection setAttribute failed: %s", errorMsg.c_str()); ThrowStdException(errorMsg); } catch (...) { // Fallback to generic error if detailed error retrieval fails std::string errorMsg = "Failed to set connection attribute " + std::to_string(attribute); - LOG("Connection setAttribute failed: {}", errorMsg); + LOG("Connection setAttribute failed: %s", errorMsg.c_str()); ThrowStdException(errorMsg); } } diff --git a/mssql_python/pybind/connection/connection_pool.cpp b/mssql_python/pybind/connection/connection_pool.cpp index cc2c4825..010676a4 100644 --- a/mssql_python/pybind/connection/connection_pool.cpp +++ b/mssql_python/pybind/connection/connection_pool.cpp @@ -6,6 +6,9 @@ #include #include +// Logging uses LOG() macro for all diagnostic output +#include "logger_bridge.hpp" + ConnectionPool::ConnectionPool(size_t max_size, int idle_timeout_secs) : _max_size(max_size), _idle_timeout_secs(idle_timeout_secs), _current_size(0) {} @@ -69,7 +72,7 @@ std::shared_ptr ConnectionPool::acquire( try { conn->disconnect(); } catch (const std::exception& ex) { - LOG("Disconnect bad/expired connections failed: {}", ex.what()); + LOG("Disconnect bad/expired connections failed: %s", ex.what()); } } return valid_conn; @@ -100,8 +103,7 @@ void ConnectionPool::close() { try { conn->disconnect(); } catch (const std::exception& ex) { - LOG("ConnectionPool::close: disconnect failed: {}", - ex.what()); + LOG("ConnectionPool::close: disconnect failed: %s", ex.what()); } } } diff --git a/mssql_python/pybind/ddbc_bindings.cpp b/mssql_python/pybind/ddbc_bindings.cpp index 96a8d9f7..f224b994 100644 --- a/mssql_python/pybind/ddbc_bindings.cpp +++ b/mssql_python/pybind/ddbc_bindings.cpp @@ -6,6 +6,7 @@ #include "ddbc_bindings.h" #include "connection/connection.h" #include "connection/connection_pool.h" +#include "logger_bridge.hpp" #include #include // std::setw, std::setfill @@ -34,6 +35,15 @@ #endif #define DAE_CHUNK_SIZE 8192 #define SQL_MAX_LOB_SIZE 8000 + +//------------------------------------------------------------------------------------------------- +//------------------------------------------------------------------------------------------------- +// Logging Infrastructure: +// - LOG() macro: All diagnostic/debug logging at DEBUG level (single level) +// - LOG_INFO/WARNING/ERROR: Higher-level messages for production +// Uses printf-style formatting: LOG("Value: %d", x) -- __FILE__/__LINE__ embedded in macro +//------------------------------------------------------------------------------------------------- + //------------------------------------------------------------------------------------------------- // Class definitions //------------------------------------------------------------------------------------------------- @@ -249,11 +259,14 @@ std::string DescribeChar(unsigned char ch) { SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, std::vector& paramInfos, std::vector>& paramBuffers) { - LOG("Starting parameter binding. Number of parameters: {}", params.size()); + LOG("BindParameters: Starting parameter binding for statement handle %p with %zu parameters", + (void*)hStmt, params.size()); for (int paramIndex = 0; paramIndex < params.size(); paramIndex++) { const auto& param = params[paramIndex]; ParamInfo& paramInfo = paramInfos[paramIndex]; - LOG("Binding parameter {} - C Type: {}, SQL Type: {}", paramIndex, paramInfo.paramCType, paramInfo.paramSQLType); + LOG("BindParameters: Processing param[%d] - C_Type=%d, SQL_Type=%d, ColumnSize=%lu, DecimalDigits=%d, InputOutputType=%d", + paramIndex, paramInfo.paramCType, paramInfo.paramSQLType, (unsigned long)paramInfo.columnSize, + paramInfo.decimalDigits, paramInfo.inputOutputType); void* dataPtr = nullptr; SQLLEN bufferLength = 0; SQLLEN* strLenOrIndPtr = nullptr; @@ -266,7 +279,7 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, ThrowStdException(MakeParamMismatchErrorStr(paramInfo.paramCType, paramIndex)); } if (paramInfo.isDAE) { - LOG("Parameter[{}] is marked for DAE streaming", paramIndex); + LOG("BindParameters: param[%d] SQL_C_CHAR - Using DAE (Data-At-Execution) for large string streaming", paramIndex); dataPtr = const_cast(reinterpret_cast(¶mInfos[paramIndex])); strLenOrIndPtr = AllocateParamBuffer(paramBuffers); *strLenOrIndPtr = SQL_LEN_DATA_AT_EXEC(0); @@ -288,7 +301,7 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, } if (paramInfo.isDAE) { // Deferred execution for VARBINARY(MAX) - LOG("Parameter[{}] is marked for DAE streaming (VARBINARY(MAX))", paramIndex); + LOG("BindParameters: param[%d] SQL_C_BINARY - Using DAE for VARBINARY(MAX) streaming", paramIndex); dataPtr = const_cast(reinterpret_cast(¶mInfos[paramIndex])); strLenOrIndPtr = AllocateParamBuffer(paramBuffers); *strLenOrIndPtr = SQL_LEN_DATA_AT_EXEC(0); @@ -318,7 +331,7 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, } if (paramInfo.isDAE) { // deferred execution - LOG("Parameter[{}] is marked for DAE streaming", paramIndex); + LOG("BindParameters: param[%d] SQL_C_WCHAR - Using DAE for NVARCHAR(MAX) streaming", paramIndex); dataPtr = const_cast(reinterpret_cast(¶mInfos[paramIndex])); strLenOrIndPtr = AllocateParamBuffer(paramBuffers); *strLenOrIndPtr = SQL_LEN_DATA_AT_EXEC(0); @@ -327,7 +340,8 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, // Normal small-string case std::wstring* strParam = AllocateParamBuffer(paramBuffers, param.cast()); - LOG("SQL_C_WCHAR Parameter[{}]: Length={}, isDAE={}", paramIndex, strParam->size(), paramInfo.isDAE); + LOG("BindParameters: param[%d] SQL_C_WCHAR - String length=%zu characters, buffer=%zu bytes", + paramIndex, strParam->size(), strParam->size() * sizeof(SQLWCHAR)); std::vector* sqlwcharBuffer = AllocateParamBuffer>(paramBuffers, WStringToSQLWCHAR(*strParam)); dataPtr = sqlwcharBuffer->data(); @@ -367,7 +381,7 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, &nullable ); if (!SQL_SUCCEEDED(rc)) { - LOG("SQLDescribeParam failed for parameter {} with error code {}", paramIndex, rc); + LOG("BindParameters: SQLDescribeParam failed for param[%d] (NULL parameter) - SQLRETURN=%d", paramIndex, rc); return rc; } sqlType = describedType; @@ -555,9 +569,8 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, ThrowStdException(MakeParamMismatchErrorStr(paramInfo.paramCType, paramIndex)); } NumericData decimalParam = param.cast(); - LOG("Received numeric parameter: precision - {}, scale- {}, sign - {}, value - {}", - decimalParam.precision, decimalParam.scale, decimalParam.sign, - decimalParam.val); + LOG("BindParameters: param[%d] SQL_C_NUMERIC - precision=%d, scale=%d, sign=%d, value_bytes=%zu", + paramIndex, decimalParam.precision, decimalParam.scale, decimalParam.sign, decimalParam.val.size()); SQL_NUMERIC_STRUCT* decimalPtr = AllocateParamBuffer(paramBuffers); decimalPtr->precision = decimalParam.precision; @@ -579,7 +592,8 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, py::bytes uuid_bytes = param.cast(); const unsigned char* uuid_data = reinterpret_cast(PyBytes_AS_STRING(uuid_bytes.ptr())); if (PyBytes_GET_SIZE(uuid_bytes.ptr()) != 16) { - LOG("Invalid UUID parameter at index {}: expected 16 bytes, got {} bytes, type {}", paramIndex, PyBytes_GET_SIZE(uuid_bytes.ptr()), paramInfo.paramCType); + LOG("BindParameters: param[%d] SQL_C_GUID - Invalid UUID length: expected 16 bytes, got %ld bytes", + paramIndex, PyBytes_GET_SIZE(uuid_bytes.ptr())); ThrowStdException("UUID binary data must be exactly 16 bytes long."); } SQLGUID* guid_data_ptr = AllocateParamBuffer(paramBuffers); @@ -617,7 +631,8 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, static_cast(paramInfo.paramSQLType), paramInfo.columnSize, paramInfo.decimalDigits, dataPtr, bufferLength, strLenOrIndPtr); if (!SQL_SUCCEEDED(rc)) { - LOG("Error when binding parameter - {}", paramIndex); + LOG("BindParameters: SQLBindParameter failed for param[%d] - SQLRETURN=%d, C_Type=%d, SQL_Type=%d", + paramIndex, rc, paramInfo.paramCType, paramInfo.paramSQLType); return rc; } // Special handling for Numeric type - @@ -626,37 +641,38 @@ SQLRETURN BindParameters(SQLHANDLE hStmt, const py::list& params, SQLHDESC hDesc = nullptr; rc = SQLGetStmtAttr_ptr(hStmt, SQL_ATTR_APP_PARAM_DESC, &hDesc, 0, NULL); if(!SQL_SUCCEEDED(rc)) { - LOG("Error when getting statement attribute - {}", paramIndex); + LOG("BindParameters: SQLGetStmtAttr(SQL_ATTR_APP_PARAM_DESC) failed for param[%d] - SQLRETURN=%d", paramIndex, rc); return rc; } rc = SQLSetDescField_ptr(hDesc, 1, SQL_DESC_TYPE, (SQLPOINTER) SQL_C_NUMERIC, 0); if(!SQL_SUCCEEDED(rc)) { - LOG("Error when setting descriptor field SQL_DESC_TYPE - {}", paramIndex); + LOG("BindParameters: SQLSetDescField(SQL_DESC_TYPE) failed for param[%d] - SQLRETURN=%d", paramIndex, rc); return rc; } SQL_NUMERIC_STRUCT* numericPtr = reinterpret_cast(dataPtr); rc = SQLSetDescField_ptr(hDesc, 1, SQL_DESC_PRECISION, (SQLPOINTER) numericPtr->precision, 0); if(!SQL_SUCCEEDED(rc)) { - LOG("Error when setting descriptor field SQL_DESC_PRECISION - {}", paramIndex); + LOG("BindParameters: SQLSetDescField(SQL_DESC_PRECISION) failed for param[%d] - SQLRETURN=%d", paramIndex, rc); return rc; } rc = SQLSetDescField_ptr(hDesc, 1, SQL_DESC_SCALE, (SQLPOINTER) numericPtr->scale, 0); if(!SQL_SUCCEEDED(rc)) { - LOG("Error when setting descriptor field SQL_DESC_SCALE - {}", paramIndex); + LOG("BindParameters: SQLSetDescField(SQL_DESC_SCALE) failed for param[%d] - SQLRETURN=%d", paramIndex, rc); return rc; } rc = SQLSetDescField_ptr(hDesc, 1, SQL_DESC_DATA_PTR, (SQLPOINTER) numericPtr, 0); if(!SQL_SUCCEEDED(rc)) { - LOG("Error when setting descriptor field SQL_DESC_DATA_PTR - {}", paramIndex); + LOG("BindParameters: SQLSetDescField(SQL_DESC_DATA_PTR) failed for param[%d] - SQLRETURN=%d", paramIndex, rc); return rc; } } } - LOG("Finished parameter binding. Number of parameters: {}", params.size()); + LOG("BindParameters: Completed parameter binding for statement handle %p - %zu parameters bound successfully", + (void*)hStmt, params.size()); return SQL_SUCCESS; } @@ -702,42 +718,6 @@ static bool is_python_finalizing() { } } -// TODO: Revisit GIL considerations if we're using python's logger -template -void LOG(const std::string& formatString, Args&&... args) { - // Check if Python is shutting down to avoid crash during cleanup - if (is_python_finalizing()) { - return; // Python is shutting down or finalizing, don't log - } - - try { - py::gil_scoped_acquire gil; // <---- this ensures safe Python API usage - - py::object logger = py::module_::import("mssql_python.logging_config").attr("get_logger")(); - if (py::isinstance(logger)) return; - - try { - std::string ddbcFormatString = "[DDBC Bindings log] " + formatString; - if constexpr (sizeof...(args) == 0) { - logger.attr("debug")(py::str(ddbcFormatString)); - } else { - py::str message = py::str(ddbcFormatString).format(std::forward(args)...); - logger.attr("debug")(message); - } - } catch (const std::exception& e) { - std::cerr << "Logging error: " << e.what() << std::endl; - } - } catch (const py::error_already_set& e) { - // Python is shutting down or in an inconsistent state, silently ignore - (void)e; // Suppress unused variable warning - return; - } catch (const std::exception& e) { - // Any other error, ignore to prevent crash during cleanup - (void)e; // Suppress unused variable warning - return; - } -} - // TODO: Add more nuanced exception classes void ThrowStdException(const std::string& message) { throw std::runtime_error(message); } std::string GetLastErrorMessage(); @@ -753,7 +733,8 @@ std::string GetModuleDirectory() { char path[MAX_PATH]; errno_t err = strncpy_s(path, MAX_PATH, module_file.c_str(), module_file.length()); if (err != 0) { - LOG("strncpy_s failed with error code: {}", err); + LOG("GetModuleDirectory: strncpy_s failed copying path - error_code=%d, path_length=%zu", + err, module_file.length()); return {}; } PathRemoveFileSpecA(path); @@ -765,21 +746,22 @@ std::string GetModuleDirectory() { std::string dir = module_file.substr(0, pos); return dir; } - LOG("DEBUG: Could not extract directory from path: {}", module_file); + LOG("GetModuleDirectory: Could not extract directory from module path - path='%s'", module_file.c_str()); return module_file; #endif } // Platform-agnostic function to load the driver dynamic library DriverHandle LoadDriverLibrary(const std::string& driverPath) { - LOG("Loading driver from path: {}", driverPath); + LOG("LoadDriverLibrary: Attempting to load ODBC driver from path='%s'", driverPath.c_str()); #ifdef _WIN32 // Windows: Convert string to wide string for LoadLibraryW std::wstring widePath(driverPath.begin(), driverPath.end()); HMODULE handle = LoadLibraryW(widePath.c_str()); if (!handle) { - LOG("Failed to load library: {}. Error: {}", driverPath, GetLastErrorMessage()); + LOG("LoadDriverLibrary: LoadLibraryW failed for path='%s' - %s", + driverPath.c_str(), GetLastErrorMessage().c_str()); ThrowStdException("Failed to load library: " + driverPath); } return handle; @@ -787,7 +769,8 @@ DriverHandle LoadDriverLibrary(const std::string& driverPath) { // macOS/Unix: Use dlopen void* handle = dlopen(driverPath.c_str(), RTLD_LAZY); if (!handle) { - LOG("dlopen failed."); + LOG("LoadDriverLibrary: dlopen failed for path='%s' - %s", + driverPath.c_str(), dlerror() ? dlerror() : "unknown error"); } return handle; #endif @@ -886,10 +869,10 @@ DriverHandle LoadDriverOrThrowException() { namespace fs = std::filesystem; std::string moduleDir = GetModuleDirectory(); - LOG("Module directory: {}", moduleDir); + LOG("LoadDriverOrThrowException: Module directory resolved to '%s'", moduleDir.c_str()); std::string archStr = ARCHITECTURE; - LOG("Architecture: {}", archStr); + LOG("LoadDriverOrThrowException: Architecture detected as '%s'", archStr.c_str()); // Use only C++ function for driver path resolution // Not using Python function since it causes circular import issues on Alpine Linux @@ -898,7 +881,7 @@ DriverHandle LoadDriverOrThrowException() { fs::path driverPath(driverPathStr); - LOG("Driver path determined: {}", driverPath.string()); + LOG("LoadDriverOrThrowException: ODBC driver path determined - path='%s'", driverPath.string().c_str()); #ifdef _WIN32 // On Windows, optionally load mssql-auth.dll if it exists @@ -912,13 +895,15 @@ DriverHandle LoadDriverOrThrowException() { if (fs::exists(authDllPath)) { HMODULE hAuth = LoadLibraryW(std::wstring(authDllPath.native().begin(), authDllPath.native().end()).c_str()); if (hAuth) { - LOG("mssql-auth.dll loaded: {}", authDllPath.string()); + LOG("LoadDriverOrThrowException: mssql-auth.dll loaded successfully from '%s'", authDllPath.string().c_str()); } else { - LOG("Failed to load mssql-auth.dll: {}", GetLastErrorMessage()); + LOG("LoadDriverOrThrowException: Failed to load mssql-auth.dll from '%s' - %s", + authDllPath.string().c_str(), GetLastErrorMessage().c_str()); ThrowStdException("Failed to load mssql-auth.dll. Please ensure it is present in the expected directory."); } } else { - LOG("Note: mssql-auth.dll not found. This is OK if Entra ID is not in use."); + LOG("LoadDriverOrThrowException: mssql-auth.dll not found at '%s' - Entra ID authentication will not be available", + authDllPath.string().c_str()); ThrowStdException("mssql-auth.dll not found. If you are using Entra ID, please ensure it is present."); } #endif @@ -929,10 +914,11 @@ DriverHandle LoadDriverOrThrowException() { DriverHandle handle = LoadDriverLibrary(driverPath.string()); if (!handle) { - LOG("Failed to load driver: {}", GetLastErrorMessage()); + LOG("LoadDriverOrThrowException: Failed to load ODBC driver - path='%s', error='%s'", + driverPath.string().c_str(), GetLastErrorMessage().c_str()); ThrowStdException("Failed to load the driver. Please read the documentation (https://github.com/microsoft/mssql-python#installation) to install the required dependencies."); } - LOG("Driver library successfully loaded."); + LOG("LoadDriverOrThrowException: ODBC driver library loaded successfully from '%s'", driverPath.string().c_str()); // Load function pointers using helper SQLAllocHandle_ptr = GetFunctionPointer(handle, "SQLAllocHandle"); @@ -999,7 +985,7 @@ DriverHandle LoadDriverOrThrowException() { if (!success) { ThrowStdException("Failed to load required function pointers from driver."); } - LOG("All driver function pointers successfully loaded."); + LOG("LoadDriverOrThrowException: All %d ODBC function pointers loaded successfully", 44); return handle; } @@ -1308,10 +1294,10 @@ SQLRETURN SQLColumns_wrap(SqlHandlePtr StatementHandle, // Helper function to check for driver errors ErrorInfo SQLCheckError_Wrap(SQLSMALLINT handleType, SqlHandlePtr handle, SQLRETURN retcode) { - LOG("Checking errors for retcode - {}" , retcode); + LOG("SQLCheckError: Checking ODBC errors - handleType=%d, retcode=%d", handleType, retcode); ErrorInfo errorInfo; if (retcode == SQL_INVALID_HANDLE) { - LOG("Invalid handle received"); + LOG("SQLCheckError: SQL_INVALID_HANDLE detected - handle is invalid"); errorInfo.ddbcErrorMsg = std::wstring( L"Invalid handle!"); return errorInfo; } @@ -1319,7 +1305,7 @@ ErrorInfo SQLCheckError_Wrap(SQLSMALLINT handleType, SqlHandlePtr handle, SQLRET SQLHANDLE rawHandle = handle->get(); if (!SQL_SUCCEEDED(retcode)) { if (!SQLGetDiagRec_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLCheckError: SQLGetDiagRec function pointer not initialized, loading driver"); DriverLoader::getInstance().loadDriver(); // Load the driver } @@ -1347,9 +1333,10 @@ ErrorInfo SQLCheckError_Wrap(SQLSMALLINT handleType, SqlHandlePtr handle, SQLRET } py::list SQLGetAllDiagRecords(SqlHandlePtr handle) { - LOG("Retrieving all diagnostic records"); + LOG("SQLGetAllDiagRecords: Retrieving all diagnostic records for handle %p, handleType=%d", + (void*)handle->get(), handle->type()); if (!SQLGetDiagRec_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLGetAllDiagRecords: SQLGetDiagRec function pointer not initialized, loading driver"); DriverLoader::getInstance().loadDriver(); } @@ -1413,9 +1400,11 @@ py::list SQLGetAllDiagRecords(SqlHandlePtr handle) { // Wrap SQLExecDirect SQLRETURN SQLExecDirect_wrap(SqlHandlePtr StatementHandle, const std::wstring& Query) { - LOG("Execute SQL query directly - {}", Query.c_str()); + std::string queryUtf8 = WideToUTF8(Query); + LOG("SQLExecDirect: Executing query directly - statement_handle=%p, query_length=%zu chars", + (void*)StatementHandle->get(), Query.length()); if (!SQLExecDirect_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLExecDirect: Function pointer not initialized, loading driver"); DriverLoader::getInstance().loadDriver(); // Load the driver } @@ -1440,7 +1429,7 @@ SQLRETURN SQLExecDirect_wrap(SqlHandlePtr StatementHandle, const std::wstring& Q #endif SQLRETURN ret = SQLExecDirect_ptr(StatementHandle->get(), queryPtr, SQL_NTS); if (!SQL_SUCCEEDED(ret)) { - LOG("Failed to execute query directly"); + LOG("SQLExecDirect: Query execution failed - SQLRETURN=%d", ret); } return ret; } @@ -1453,7 +1442,7 @@ SQLRETURN SQLTables_wrap(SqlHandlePtr StatementHandle, const std::wstring& tableType) { if (!SQLTables_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLTables: Function pointer not initialized, loading driver"); DriverLoader::getInstance().loadDriver(); } @@ -1521,11 +1510,8 @@ SQLRETURN SQLTables_wrap(SqlHandlePtr StatementHandle, tableTypePtr, tableTypeLen ); - if (!SQL_SUCCEEDED(ret)) { - LOG("SQLTables failed with return code: {}", ret); - } else { - LOG("SQLTables succeeded"); - } + LOG("SQLTables: Catalog metadata query %s - SQLRETURN=%d", + SQL_SUCCEEDED(ret) ? "succeeded" : "failed", ret); return ret; } @@ -1538,9 +1524,10 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, const std::wstring& query /* TODO: Use SQLTCHAR? */, const py::list& params, std::vector& paramInfos, py::list& isStmtPrepared, const bool usePrepare = true) { - LOG("Execute SQL Query - {}", query.c_str()); + LOG("SQLExecute: Executing %s query - statement_handle=%p, param_count=%zu, query_length=%zu chars", + (params.size() > 0 ? "parameterized" : "direct"), (void*)statementHandle->get(), params.size(), query.length()); if (!SQLPrepare_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLExecute: Function pointer not initialized, loading driver"); DriverLoader::getInstance().loadDriver(); // Load the driver } assert(SQLPrepare_ptr && SQLBindParameter_ptr && SQLExecute_ptr && SQLExecDirect_ptr); @@ -1553,7 +1540,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, RETCODE rc; SQLHANDLE hStmt = statementHandle->get(); if (!statementHandle || !statementHandle->get()) { - LOG("Statement handle is null or empty"); + LOG("SQLExecute: Statement handle is null or invalid"); } // Ensure statement is scrollable BEFORE executing @@ -1582,7 +1569,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, // https://learn.microsoft.com/en-us/sql/odbc/reference/syntax/sqlexecdirect-function?view=sql-server-ver16 rc = SQLExecDirect_ptr(hStmt, queryPtr, SQL_NTS); if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) { - LOG("Error during direct execution of the statement"); + LOG("SQLExecute: Direct execution failed (non-parameterized query) - SQLRETURN=%d", rc); } return rc; } else { @@ -1593,7 +1580,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, if (usePrepare) { rc = SQLPrepare_ptr(hStmt, queryPtr, SQL_NTS); if (!SQL_SUCCEEDED(rc)) { - LOG("Error while preparing the statement"); + LOG("SQLExecute: SQLPrepare failed - SQLRETURN=%d, statement_handle=%p", rc, (void*)hStmt); return rc; } isStmtPrepared[0] = py::cast(true); @@ -1616,7 +1603,7 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, rc = SQLExecute_ptr(hStmt); if (rc == SQL_NEED_DATA) { - LOG("Beginning SQLParamData/SQLPutData loop for DAE."); + LOG("SQLExecute: SQL_NEED_DATA received - Starting DAE (Data-At-Execution) loop for large parameter streaming"); SQLPOINTER paramToken = nullptr; while ((rc = SQLParamData_ptr(hStmt, ¶mToken)) == SQL_NEED_DATA) { // Finding the paramInfo that matches the returned token @@ -1658,7 +1645,8 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, } rc = SQLPutData_ptr(hStmt, (SQLPOINTER)(dataPtr + offset), static_cast(lenBytes)); if (!SQL_SUCCEEDED(rc)) { - LOG("SQLPutData failed at offset {} of {}", offset, totalChars); + LOG("SQLExecute: SQLPutData failed for SQL_C_WCHAR chunk - offset=%zu, total_chars=%zu, chunk_bytes=%zu, SQLRETURN=%d", + offset, totalChars, lenBytes, rc); return rc; } offset += len; @@ -1674,7 +1662,8 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, rc = SQLPutData_ptr(hStmt, (SQLPOINTER)(dataPtr + offset), static_cast(len)); if (!SQL_SUCCEEDED(rc)) { - LOG("SQLPutData failed at offset {} of {}", offset, totalBytes); + LOG("SQLExecute: SQLPutData failed for SQL_C_CHAR chunk - offset=%zu, total_bytes=%zu, chunk_bytes=%zu, SQLRETURN=%d", + offset, totalBytes, len, rc); return rc; } offset += len; @@ -1692,7 +1681,8 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, size_t len = std::min(chunkSize, totalBytes - offset); rc = SQLPutData_ptr(hStmt, (SQLPOINTER)(dataPtr + offset), static_cast(len)); if (!SQL_SUCCEEDED(rc)) { - LOG("SQLPutData failed at offset {} of {}", offset, totalBytes); + LOG("SQLExecute: SQLPutData failed for binary/bytes chunk - offset=%zu, total_bytes=%zu, chunk_bytes=%zu, SQLRETURN=%d", + offset, totalBytes, len, rc); return rc; } } @@ -1701,13 +1691,14 @@ SQLRETURN SQLExecute_wrap(const SqlHandlePtr statementHandle, } } if (!SQL_SUCCEEDED(rc)) { - LOG("SQLParamData final rc: {}", rc); + LOG("SQLExecute: SQLParamData final call %s - SQLRETURN=%d", + (rc == SQL_NO_DATA ? "completed with no data" : "failed"), rc); return rc; } - LOG("DAE complete, SQLExecute resumed internally."); + LOG("SQLExecute: DAE streaming completed successfully, SQLExecute resumed"); } if (!SQL_SUCCEEDED(rc) && rc != SQL_NO_DATA) { - LOG("DDBCSQLExecute: Error during execution of the statement"); + LOG("SQLExecute: Statement execution failed - SQLRETURN=%d, statement_handle=%p", rc, (void*)hStmt); return rc; } @@ -1723,7 +1714,8 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, const std::vector& paramInfos, size_t paramSetSize, std::vector>& paramBuffers) { - LOG("Starting column-wise parameter array binding. paramSetSize: {}, paramCount: {}", paramSetSize, columnwise_params.size()); + LOG("BindParameterArray: Starting column-wise array binding - param_count=%zu, param_set_size=%zu", + columnwise_params.size(), paramSetSize); std::vector> tempBuffers; @@ -1731,7 +1723,11 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, for (int paramIndex = 0; paramIndex < columnwise_params.size(); ++paramIndex) { const py::list& columnValues = columnwise_params[paramIndex].cast(); const ParamInfo& info = paramInfos[paramIndex]; + LOG("BindParameterArray: Processing param_index=%d, C_type=%d, SQL_type=%d, column_size=%zu, decimal_digits=%d", + paramIndex, info.paramCType, info.paramSQLType, info.columnSize, info.decimalDigits); if (columnValues.size() != paramSetSize) { + LOG("BindParameterArray: Size mismatch - param_index=%d, expected=%zu, actual=%zu", + paramIndex, paramSetSize, columnValues.size()); ThrowStdException("Column " + std::to_string(paramIndex) + " has mismatched size."); } void* dataPtr = nullptr; @@ -1739,54 +1735,70 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, SQLLEN bufferLength = 0; switch (info.paramCType) { case SQL_C_LONG: { + LOG("BindParameterArray: Binding SQL_C_LONG array - param_index=%d, count=%zu", paramIndex, paramSetSize); int* dataArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { if (!strLenOrIndArray) strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); dataArray[i] = 0; strLenOrIndArray[i] = SQL_NULL_DATA; + null_count++; } else { dataArray[i] = columnValues[i].cast(); if (strLenOrIndArray) strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_LONG bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = dataArray; break; } case SQL_C_DOUBLE: { + LOG("BindParameterArray: Binding SQL_C_DOUBLE array - param_index=%d, count=%zu", paramIndex, paramSetSize); double* dataArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { if (!strLenOrIndArray) strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); dataArray[i] = 0; strLenOrIndArray[i] = SQL_NULL_DATA; + null_count++; } else { dataArray[i] = columnValues[i].cast(); if (strLenOrIndArray) strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_DOUBLE bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = dataArray; break; } case SQL_C_WCHAR: { + LOG("BindParameterArray: Binding SQL_C_WCHAR array - param_index=%d, count=%zu, column_size=%zu", + paramIndex, paramSetSize, info.columnSize); SQLWCHAR* wcharArray = AllocateParamBufferArray(tempBuffers, paramSetSize * (info.columnSize + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0, total_chars = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; std::memset(wcharArray + i * (info.columnSize + 1), 0, (info.columnSize + 1) * sizeof(SQLWCHAR)); + null_count++; } else { std::wstring wstr = columnValues[i].cast(); #if defined(__APPLE__) || defined(__linux__) // Convert to UTF-16 first, then check the actual UTF-16 length auto utf16Buf = WStringToSQLWCHAR(wstr); + size_t utf16_len = utf16Buf.size() > 0 ? utf16Buf.size() - 1 : 0; + total_chars += utf16_len; // Check UTF-16 length (excluding null terminator) against column size - if (utf16Buf.size() > 0 && (utf16Buf.size() - 1) > info.columnSize) { + if (utf16Buf.size() > 0 && utf16_len > info.columnSize) { std::string offending = WideToUTF8(wstr); + LOG("BindParameterArray: SQL_C_WCHAR string too long - param_index=%d, row=%zu, utf16_length=%zu, max=%zu", + paramIndex, i, utf16_len, info.columnSize); ThrowStdException("Input string UTF-16 length exceeds allowed column size at parameter index " + std::to_string(paramIndex) + - ". UTF-16 length: " + std::to_string(utf16Buf.size() - 1) + ", Column size: " + std::to_string(info.columnSize)); + ". UTF-16 length: " + std::to_string(utf16_len) + ", Column size: " + std::to_string(info.columnSize)); } // If we reach here, the UTF-16 string fits - copy it completely std::memcpy(wcharArray + i * (info.columnSize + 1), utf16Buf.data(), utf16Buf.size() * sizeof(SQLWCHAR)); @@ -1801,103 +1813,138 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, strLenOrIndArray[i] = SQL_NTS; } } + LOG("BindParameterArray: SQL_C_WCHAR bound - param_index=%d, null_values=%zu, total_chars=%zu", + paramIndex, null_count, total_chars); dataPtr = wcharArray; bufferLength = (info.columnSize + 1) * sizeof(SQLWCHAR); break; } case SQL_C_TINYINT: case SQL_C_UTINYINT: { + LOG("BindParameterArray: Binding SQL_C_TINYINT/UTINYINT array - param_index=%d, count=%zu", paramIndex, paramSetSize); unsigned char* dataArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { if (!strLenOrIndArray) strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); dataArray[i] = 0; strLenOrIndArray[i] = SQL_NULL_DATA; + null_count++; } else { int intVal = columnValues[i].cast(); if (intVal < 0 || intVal > 255) { + LOG("BindParameterArray: TINYINT value out of range - param_index=%d, row=%zu, value=%d", + paramIndex, i, intVal); ThrowStdException("UTINYINT value out of range at rowIndex " + std::to_string(i)); } dataArray[i] = static_cast(intVal); if (strLenOrIndArray) strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_TINYINT bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = dataArray; bufferLength = sizeof(unsigned char); break; } case SQL_C_SHORT: { + LOG("BindParameterArray: Binding SQL_C_SHORT array - param_index=%d, count=%zu", paramIndex, paramSetSize); short* dataArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { if (!strLenOrIndArray) strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); dataArray[i] = 0; strLenOrIndArray[i] = SQL_NULL_DATA; + null_count++; } else { int intVal = columnValues[i].cast(); if (intVal < std::numeric_limits::min() || intVal > std::numeric_limits::max()) { + LOG("BindParameterArray: SHORT value out of range - param_index=%d, row=%zu, value=%d", + paramIndex, i, intVal); ThrowStdException("SHORT value out of range at rowIndex " + std::to_string(i)); } dataArray[i] = static_cast(intVal); if (strLenOrIndArray) strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_SHORT bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = dataArray; bufferLength = sizeof(short); break; } case SQL_C_CHAR: case SQL_C_BINARY: { + LOG("BindParameterArray: Binding SQL_C_CHAR/BINARY array - param_index=%d, count=%zu, column_size=%zu", + paramIndex, paramSetSize, info.columnSize); char* charArray = AllocateParamBufferArray(tempBuffers, paramSetSize * (info.columnSize + 1)); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0, total_bytes = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; std::memset(charArray + i * (info.columnSize + 1), 0, info.columnSize + 1); + null_count++; } else { std::string str = columnValues[i].cast(); - if (str.size() > info.columnSize) + total_bytes += str.size(); + if (str.size() > info.columnSize) { + LOG("BindParameterArray: String/binary too long - param_index=%d, row=%zu, size=%zu, max=%zu", + paramIndex, i, str.size(), info.columnSize); ThrowStdException("Input exceeds column size at index " + std::to_string(i)); + } std::memcpy(charArray + i * (info.columnSize + 1), str.c_str(), str.size()); strLenOrIndArray[i] = static_cast(str.size()); } } + LOG("BindParameterArray: SQL_C_CHAR/BINARY bound - param_index=%d, null_values=%zu, total_bytes=%zu", + paramIndex, null_count, total_bytes); dataPtr = charArray; bufferLength = info.columnSize + 1; break; } case SQL_C_BIT: { + LOG("BindParameterArray: Binding SQL_C_BIT array - param_index=%d, count=%zu", paramIndex, paramSetSize); char* boolArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0, true_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { boolArray[i] = 0; strLenOrIndArray[i] = SQL_NULL_DATA; + null_count++; } else { - boolArray[i] = columnValues[i].cast() ? 1 : 0; + bool val = columnValues[i].cast(); + boolArray[i] = val ? 1 : 0; + if (val) true_count++; strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_BIT bound - param_index=%d, null_values=%zu, true_values=%zu", + paramIndex, null_count, true_count); dataPtr = boolArray; bufferLength = sizeof(char); break; } case SQL_C_STINYINT: case SQL_C_USHORT: { + LOG("BindParameterArray: Binding SQL_C_USHORT/STINYINT array - param_index=%d, count=%zu", paramIndex, paramSetSize); unsigned short* dataArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; dataArray[i] = 0; + null_count++; } else { dataArray[i] = columnValues[i].cast(); strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_USHORT bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = dataArray; bufferLength = sizeof(unsigned short); break; @@ -1906,44 +1953,55 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, case SQL_C_SLONG: case SQL_C_UBIGINT: case SQL_C_ULONG: { + LOG("BindParameterArray: Binding SQL_C_BIGINT array - param_index=%d, count=%zu", paramIndex, paramSetSize); int64_t* dataArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; dataArray[i] = 0; + null_count++; } else { dataArray[i] = columnValues[i].cast(); strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_BIGINT bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = dataArray; bufferLength = sizeof(int64_t); break; } case SQL_C_FLOAT: { + LOG("BindParameterArray: Binding SQL_C_FLOAT array - param_index=%d, count=%zu", paramIndex, paramSetSize); float* dataArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; dataArray[i] = 0.0f; + null_count++; } else { dataArray[i] = columnValues[i].cast(); strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_FLOAT bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = dataArray; bufferLength = sizeof(float); break; } case SQL_C_TYPE_DATE: { + LOG("BindParameterArray: Binding SQL_C_TYPE_DATE array - param_index=%d, count=%zu", paramIndex, paramSetSize); SQL_DATE_STRUCT* dateArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; std::memset(&dateArray[i], 0, sizeof(SQL_DATE_STRUCT)); + null_count++; } else { py::object dateObj = columnValues[i]; dateArray[i].year = dateObj.attr("year").cast(); @@ -1952,17 +2010,21 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_TYPE_DATE bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = dateArray; bufferLength = sizeof(SQL_DATE_STRUCT); break; } case SQL_C_TYPE_TIME: { + LOG("BindParameterArray: Binding SQL_C_TYPE_TIME array - param_index=%d, count=%zu", paramIndex, paramSetSize); SQL_TIME_STRUCT* timeArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; std::memset(&timeArray[i], 0, sizeof(SQL_TIME_STRUCT)); + null_count++; } else { py::object timeObj = columnValues[i]; timeArray[i].hour = timeObj.attr("hour").cast(); @@ -1971,17 +2033,21 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_TYPE_TIME bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = timeArray; bufferLength = sizeof(SQL_TIME_STRUCT); break; } case SQL_C_TYPE_TIMESTAMP: { + LOG("BindParameterArray: Binding SQL_C_TYPE_TIMESTAMP array - param_index=%d, count=%zu", paramIndex, paramSetSize); SQL_TIMESTAMP_STRUCT* tsArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { if (columnValues[i].is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; std::memset(&tsArray[i], 0, sizeof(SQL_TIMESTAMP_STRUCT)); + null_count++; } else { py::object dtObj = columnValues[i]; tsArray[i].year = dtObj.attr("year").cast(); @@ -1994,15 +2060,18 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, strLenOrIndArray[i] = 0; } } + LOG("BindParameterArray: SQL_C_TYPE_TIMESTAMP bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = tsArray; bufferLength = sizeof(SQL_TIMESTAMP_STRUCT); break; } case SQL_C_SS_TIMESTAMPOFFSET: { + LOG("BindParameterArray: Binding SQL_C_SS_TIMESTAMPOFFSET array - param_index=%d, count=%zu", paramIndex, paramSetSize); DateTimeOffset* dtoArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); py::object datetimeType = py::module_::import("datetime").attr("datetime"); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { const py::handle& param = columnValues[i]; @@ -2010,6 +2079,7 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, if (param.is_none()) { std::memset(&dtoArray[i], 0, sizeof(DateTimeOffset)); strLenOrIndArray[i] = SQL_NULL_DATA; + null_count++; } else { if (!py::isinstance(param, datetimeType)) { ThrowStdException(MakeParamMismatchErrorStr(info.paramCType, paramIndex)); @@ -2041,26 +2111,31 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, strLenOrIndArray[i] = sizeof(DateTimeOffset); } } + LOG("BindParameterArray: SQL_C_SS_TIMESTAMPOFFSET bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = dtoArray; bufferLength = sizeof(DateTimeOffset); break; } case SQL_C_NUMERIC: { + LOG("BindParameterArray: Binding SQL_C_NUMERIC array - param_index=%d, count=%zu", paramIndex, paramSetSize); SQL_NUMERIC_STRUCT* numericArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); + size_t null_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { const py::handle& element = columnValues[i]; if (element.is_none()) { strLenOrIndArray[i] = SQL_NULL_DATA; std::memset(&numericArray[i], 0, sizeof(SQL_NUMERIC_STRUCT)); + null_count++; continue; } if (!py::isinstance(element)) { + LOG("BindParameterArray: NUMERIC type mismatch - param_index=%d, row=%zu", paramIndex, i); throw std::runtime_error(MakeParamMismatchErrorStr(info.paramCType, paramIndex)); } NumericData decimalParam = element.cast(); - LOG("Received numeric parameter at [%zu]: precision=%d, scale=%d, sign=%d, val=%s", - i, decimalParam.precision, decimalParam.scale, decimalParam.sign, decimalParam.val.c_str()); + LOG("BindParameterArray: NUMERIC value - param_index=%d, row=%zu, precision=%d, scale=%d, sign=%d", + paramIndex, i, decimalParam.precision, decimalParam.scale, decimalParam.sign); SQL_NUMERIC_STRUCT& target = numericArray[i]; std::memset(&target, 0, sizeof(SQL_NUMERIC_STRUCT)); target.precision = decimalParam.precision; @@ -2072,17 +2147,20 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, } strLenOrIndArray[i] = sizeof(SQL_NUMERIC_STRUCT); } + LOG("BindParameterArray: SQL_C_NUMERIC bound - param_index=%d, null_values=%zu", paramIndex, null_count); dataPtr = numericArray; bufferLength = sizeof(SQL_NUMERIC_STRUCT); break; } case SQL_C_GUID: { + LOG("BindParameterArray: Binding SQL_C_GUID array - param_index=%d, count=%zu", paramIndex, paramSetSize); SQLGUID* guidArray = AllocateParamBufferArray(tempBuffers, paramSetSize); strLenOrIndArray = AllocateParamBufferArray(tempBuffers, paramSetSize); // Get cached UUID class from module-level helper // This avoids static object destruction issues during Python finalization py::object uuid_class = py::module_::import("mssql_python.ddbc_bindings").attr("_get_uuid_class")(); + size_t null_count = 0, bytes_count = 0, uuid_count = 0; for (size_t i = 0; i < paramSetSize; ++i) { const py::handle& element = columnValues[i]; @@ -2090,20 +2168,26 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, if (element.is_none()) { std::memset(&guidArray[i], 0, sizeof(SQLGUID)); strLenOrIndArray[i] = SQL_NULL_DATA; + null_count++; continue; } else if (py::isinstance(element)) { py::bytes b = element.cast(); if (PyBytes_GET_SIZE(b.ptr()) != 16) { + LOG("BindParameterArray: GUID bytes wrong length - param_index=%d, row=%zu, length=%d", + paramIndex, i, PyBytes_GET_SIZE(b.ptr())); ThrowStdException("UUID binary data must be exactly 16 bytes long."); } std::memcpy(uuid_bytes.data(), PyBytes_AS_STRING(b.ptr()), 16); + bytes_count++; } else if (py::isinstance(element, uuid_class)) { py::bytes b = element.attr("bytes_le").cast(); std::memcpy(uuid_bytes.data(), PyBytes_AS_STRING(b.ptr()), 16); + uuid_count++; } else { + LOG("BindParameterArray: GUID type mismatch - param_index=%d, row=%zu", paramIndex, i); ThrowStdException(MakeParamMismatchErrorStr(info.paramCType, paramIndex)); } guidArray[i].Data1 = (static_cast(uuid_bytes[3]) << 24) | @@ -2117,14 +2201,19 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, std::memcpy(guidArray[i].Data4, uuid_bytes.data() + 8, 8); strLenOrIndArray[i] = sizeof(SQLGUID); } + LOG("BindParameterArray: SQL_C_GUID bound - param_index=%d, null=%zu, bytes=%zu, uuid_obj=%zu", + paramIndex, null_count, bytes_count, uuid_count); dataPtr = guidArray; bufferLength = sizeof(SQLGUID); break; } default: { + LOG("BindParameterArray: Unsupported C type - param_index=%d, C_type=%d", paramIndex, info.paramCType); ThrowStdException("BindParameterArray: Unsupported C type: " + std::to_string(info.paramCType)); } } + LOG("BindParameterArray: Calling SQLBindParameter - param_index=%d, buffer_length=%lld", + paramIndex, static_cast(bufferLength)); RETCODE rc = SQLBindParameter_ptr( hStmt, static_cast(paramIndex + 1), @@ -2138,16 +2227,17 @@ SQLRETURN BindParameterArray(SQLHANDLE hStmt, strLenOrIndArray ); if (!SQL_SUCCEEDED(rc)) { - LOG("Failed to bind array param {}", paramIndex); + LOG("BindParameterArray: SQLBindParameter failed - param_index=%d, SQLRETURN=%d", paramIndex, rc); return rc; } } } catch (...) { - LOG("Exception occurred during parameter array binding. Cleaning up."); + LOG("BindParameterArray: Exception during binding, cleaning up buffers"); throw; } paramBuffers.insert(paramBuffers.end(), tempBuffers.begin(), tempBuffers.end()); - LOG("Finished column-wise parameter array binding."); + LOG("BindParameterArray: Successfully bound all parameters - total_params=%zu, buffer_count=%zu", + columnwise_params.size(), paramBuffers.size()); return SQL_SUCCESS; } @@ -2156,17 +2246,25 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, const py::list& columnwise_params, const std::vector& paramInfos, size_t paramSetSize) { + LOG("SQLExecuteMany: Starting batch execution - param_count=%zu, param_set_size=%zu", + columnwise_params.size(), paramSetSize); SQLHANDLE hStmt = statementHandle->get(); SQLWCHAR* queryPtr; #if defined(__APPLE__) || defined(__linux__) std::vector queryBuffer = WStringToSQLWCHAR(query); queryPtr = queryBuffer.data(); + LOG("SQLExecuteMany: Query converted to SQLWCHAR - buffer_size=%zu", queryBuffer.size()); #else queryPtr = const_cast(query.c_str()); + LOG("SQLExecuteMany: Using wide string query directly"); #endif RETCODE rc = SQLPrepare_ptr(hStmt, queryPtr, SQL_NTS); - if (!SQL_SUCCEEDED(rc)) return rc; + if (!SQL_SUCCEEDED(rc)) { + LOG("SQLExecuteMany: SQLPrepare failed - rc=%d", rc); + return rc; + } + LOG("SQLExecuteMany: Query prepared successfully"); bool hasDAE = false; for (const auto& p : paramInfos) { @@ -2175,50 +2273,93 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, break; } } + LOG("SQLExecuteMany: Parameter analysis - hasDAE=%s", hasDAE ? "true" : "false"); if (!hasDAE) { + LOG("SQLExecuteMany: Using array binding (non-DAE) - calling BindParameterArray"); std::vector> paramBuffers; rc = BindParameterArray(hStmt, columnwise_params, paramInfos, paramSetSize, paramBuffers); - if (!SQL_SUCCEEDED(rc)) return rc; + if (!SQL_SUCCEEDED(rc)) { + LOG("SQLExecuteMany: BindParameterArray failed - rc=%d", rc); + return rc; + } rc = SQLSetStmtAttr_ptr(hStmt, SQL_ATTR_PARAMSET_SIZE, (SQLPOINTER)paramSetSize, 0); - if (!SQL_SUCCEEDED(rc)) return rc; + if (!SQL_SUCCEEDED(rc)) { + LOG("SQLExecuteMany: SQLSetStmtAttr(PARAMSET_SIZE) failed - rc=%d", rc); + return rc; + } + LOG("SQLExecuteMany: PARAMSET_SIZE set to %zu", paramSetSize); rc = SQLExecute_ptr(hStmt); + LOG("SQLExecuteMany: SQLExecute completed - rc=%d", rc); return rc; } else { + LOG("SQLExecuteMany: Using DAE (data-at-execution) - row_count=%zu", columnwise_params.size()); size_t rowCount = columnwise_params.size(); for (size_t rowIndex = 0; rowIndex < rowCount; ++rowIndex) { + LOG("SQLExecuteMany: Processing DAE row %zu of %zu", rowIndex + 1, rowCount); py::list rowParams = columnwise_params[rowIndex]; std::vector> paramBuffers; rc = BindParameters(hStmt, rowParams, const_cast&>(paramInfos), paramBuffers); - if (!SQL_SUCCEEDED(rc)) return rc; + if (!SQL_SUCCEEDED(rc)) { + LOG("SQLExecuteMany: BindParameters failed for row %zu - rc=%d", rowIndex, rc); + return rc; + } + LOG("SQLExecuteMany: Parameters bound for row %zu", rowIndex); rc = SQLExecute_ptr(hStmt); + LOG("SQLExecuteMany: SQLExecute for row %zu - initial_rc=%d", rowIndex, rc); + size_t dae_chunk_count = 0; while (rc == SQL_NEED_DATA) { SQLPOINTER token; rc = SQLParamData_ptr(hStmt, &token); - if (!SQL_SUCCEEDED(rc) && rc != SQL_NEED_DATA) return rc; + LOG("SQLExecuteMany: SQLParamData called - chunk=%zu, rc=%d, token=%p", + dae_chunk_count, rc, token); + if (!SQL_SUCCEEDED(rc) && rc != SQL_NEED_DATA) { + LOG("SQLExecuteMany: SQLParamData failed - chunk=%zu, rc=%d", dae_chunk_count, rc); + return rc; + } py::object* py_obj_ptr = reinterpret_cast(token); - if (!py_obj_ptr) return SQL_ERROR; + if (!py_obj_ptr) { + LOG("SQLExecuteMany: NULL token pointer in DAE - chunk=%zu", dae_chunk_count); + return SQL_ERROR; + } if (py::isinstance(*py_obj_ptr)) { std::string data = py_obj_ptr->cast(); SQLLEN data_len = static_cast(data.size()); + LOG("SQLExecuteMany: Sending string DAE data - chunk=%zu, length=%lld", + dae_chunk_count, static_cast(data_len)); rc = SQLPutData_ptr(hStmt, (SQLPOINTER)data.c_str(), data_len); + if (!SQL_SUCCEEDED(rc) && rc != SQL_NEED_DATA) { + LOG("SQLExecuteMany: SQLPutData(string) failed - chunk=%zu, rc=%d", dae_chunk_count, rc); + } } else if (py::isinstance(*py_obj_ptr) || py::isinstance(*py_obj_ptr)) { std::string data = py_obj_ptr->cast(); SQLLEN data_len = static_cast(data.size()); + LOG("SQLExecuteMany: Sending bytes/bytearray DAE data - chunk=%zu, length=%lld", + dae_chunk_count, static_cast(data_len)); rc = SQLPutData_ptr(hStmt, (SQLPOINTER)data.c_str(), data_len); + if (!SQL_SUCCEEDED(rc) && rc != SQL_NEED_DATA) { + LOG("SQLExecuteMany: SQLPutData(bytes) failed - chunk=%zu, rc=%d", dae_chunk_count, rc); + } } else { - LOG("Unsupported DAE parameter type in row {}", rowIndex); + LOG("SQLExecuteMany: Unsupported DAE data type - chunk=%zu", dae_chunk_count); return SQL_ERROR; } + dae_chunk_count++; } + LOG("SQLExecuteMany: DAE completed for row %zu - total_chunks=%zu, final_rc=%d", + rowIndex, dae_chunk_count, rc); - if (!SQL_SUCCEEDED(rc)) return rc; + if (!SQL_SUCCEEDED(rc)) { + LOG("SQLExecuteMany: DAE row %zu failed - rc=%d", rowIndex, rc); + return rc; + } } + LOG("SQLExecuteMany: All DAE rows processed successfully - total_rows=%zu", rowCount); return SQL_SUCCESS; } } @@ -2226,9 +2367,9 @@ SQLRETURN SQLExecuteMany_wrap(const SqlHandlePtr statementHandle, // Wrap SQLNumResultCols SQLSMALLINT SQLNumResultCols_wrap(SqlHandlePtr statementHandle) { - LOG("Get number of columns in result set"); + LOG("SQLNumResultCols: Getting number of columns in result set for statement_handle=%p", (void*)statementHandle->get()); if (!SQLNumResultCols_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLNumResultCols: Function pointer not initialized, loading driver"); DriverLoader::getInstance().loadDriver(); // Load the driver } @@ -2240,9 +2381,9 @@ SQLSMALLINT SQLNumResultCols_wrap(SqlHandlePtr statementHandle) { // Wrap SQLDescribeCol SQLRETURN SQLDescribeCol_wrap(SqlHandlePtr StatementHandle, py::list& ColumnMetadata) { - LOG("Get column description"); + LOG("SQLDescribeCol: Getting column descriptions for statement_handle=%p", (void*)StatementHandle->get()); if (!SQLDescribeCol_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLDescribeCol: Function pointer not initialized, loading driver"); DriverLoader::getInstance().loadDriver(); // Load the driver } @@ -2250,7 +2391,7 @@ SQLRETURN SQLDescribeCol_wrap(SqlHandlePtr StatementHandle, py::list& ColumnMeta SQLRETURN retcode = SQLNumResultCols_ptr(StatementHandle->get(), &ColumnCount); if (!SQL_SUCCEEDED(retcode)) { - LOG("Failed to get number of columns"); + LOG("SQLDescribeCol: Failed to get number of columns - SQLRETURN=%d", retcode); return retcode; } @@ -2334,9 +2475,9 @@ SQLRETURN SQLSpecialColumns_wrap(SqlHandlePtr StatementHandle, // Wrap SQLFetch to retrieve rows SQLRETURN SQLFetch_wrap(SqlHandlePtr StatementHandle) { - LOG("Fetch next row"); + LOG("SQLFetch: Fetching next row for statement_handle=%p", (void*)StatementHandle->get()); if (!SQLFetch_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLFetch: Function pointer not initialized, loading driver"); DriverLoader::getInstance().loadDriver(); // Load the driver } @@ -2370,11 +2511,11 @@ static py::object FetchLobColumnData(SQLHSTMT hStmt, << ", cType=" << cType << ", loop=" << loopCount << ", SQLGetData return=" << ret; - LOG(oss.str()); + LOG("FetchLobColumnData: %s", oss.str().c_str()); ThrowStdException(oss.str()); } if (actualRead == SQL_NULL_DATA) { - LOG("Loop {}: Column {} is NULL", loopCount, colIndex); + LOG("FetchLobColumnData: Column %d is NULL at loop %d", colIndex, loopCount); return py::none(); } @@ -2397,7 +2538,7 @@ static py::object FetchLobColumnData(SQLHSTMT hStmt, --bytesRead; } if (bytesRead < DAE_CHUNK_SIZE) { - LOG("Loop {}: Trimmed null terminator (narrow)", loopCount); + LOG("FetchLobColumnData: Trimmed null terminator from narrow char data - loop=%d", loopCount); } } else { // Wide characters @@ -2410,21 +2551,21 @@ static py::object FetchLobColumnData(SQLHSTMT hStmt, bytesRead -= wcharSize; } if (bytesRead < DAE_CHUNK_SIZE) { - LOG("Loop {}: Trimmed null terminator (wide)", loopCount); + LOG("FetchLobColumnData: Trimmed null terminator from wide char data - loop=%d", loopCount); } } } } if (bytesRead > 0) { buffer.insert(buffer.end(), chunk.begin(), chunk.begin() + bytesRead); - LOG("Loop {}: Appended {} bytes", loopCount, bytesRead); + LOG("FetchLobColumnData: Appended %zu bytes at loop %d", bytesRead, loopCount); } if (ret == SQL_SUCCESS) { - LOG("Loop {}: SQL_SUCCESS, no more data", loopCount); + LOG("FetchLobColumnData: SQL_SUCCESS - no more data at loop %d", loopCount); break; } } - LOG("FetchLobColumnData: Total bytes collected = {}", buffer.size()); + LOG("FetchLobColumnData: Total bytes collected=%zu for column %d", buffer.size(), colIndex); if (buffer.empty()) { if (isBinary) { @@ -2447,19 +2588,19 @@ static py::object FetchLobColumnData(SQLHSTMT hStmt, #endif } if (isBinary) { - LOG("FetchLobColumnData: Returning binary of {} bytes", buffer.size()); + LOG("FetchLobColumnData: Returning binary data - %zu bytes for column %d", buffer.size(), colIndex); return py::bytes(buffer.data(), buffer.size()); } std::string str(buffer.data(), buffer.size()); - LOG("FetchLobColumnData: Returning narrow string of length {}", str.length()); + LOG("FetchLobColumnData: Returning narrow string - length=%zu for column %d", str.length(), colIndex); return py::str(str); } // Helper function to retrieve column data SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, py::list& row) { - LOG("Get data from columns"); + LOG("SQLGetData: Getting data from %d columns for statement_handle=%p", colCount, (void*)StatementHandle->get()); if (!SQLGetData_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLGetData: Function pointer not initialized, loading driver"); DriverLoader::getInstance().loadDriver(); // Load the driver } @@ -2476,7 +2617,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p ret = SQLDescribeCol_ptr(hStmt, i, columnName, sizeof(columnName) / sizeof(SQLWCHAR), &columnNameLen, &dataType, &columnSize, &decimalDigits, &nullable); if (!SQL_SUCCEEDED(ret)) { - LOG("Error retrieving data for column - {}, SQLDescribeCol return code - {}", i, ret); + LOG("SQLGetData: Error retrieving metadata for column %d - SQLDescribeCol SQLRETURN=%d", i, ret); row.append(py::none()); continue; } @@ -2486,7 +2627,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p case SQL_VARCHAR: case SQL_LONGVARCHAR: { if (columnSize == SQL_NO_TOTAL || columnSize == 0 || columnSize > SQL_MAX_LOB_SIZE) { - LOG("Streaming LOB for column {}", i); + LOG("SQLGetData: Streaming LOB for column %d (SQL_C_CHAR) - columnSize=%lu", i, (unsigned long)columnSize); row.append(FetchLobColumnData(hStmt, i, SQL_C_CHAR, false, false)); } else { uint64_t fetchBufferSize = columnSize + 1 /* null-termination */; @@ -2503,34 +2644,28 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p #if defined(__APPLE__) || defined(__linux__) std::string fullStr(reinterpret_cast(dataBuffer.data())); row.append(fullStr); - LOG("macOS/Linux: Appended CHAR string of length {} to result row", fullStr.length()); #else row.append(std::string(reinterpret_cast(dataBuffer.data()))); #endif } else { // Buffer too small, fallback to streaming - LOG("CHAR column {} data truncated, using streaming LOB", i); + LOG("SQLGetData: CHAR column %d data truncated (buffer_size=%zu), using streaming LOB", i, dataBuffer.size()); row.append(FetchLobColumnData(hStmt, i, SQL_C_CHAR, false, false)); } } else if (dataLen == SQL_NULL_DATA) { - LOG("Column {} is NULL (CHAR)", i); + LOG("SQLGetData: Column %d is NULL (CHAR)", i); row.append(py::none()); } else if (dataLen == 0) { row.append(py::str("")); } else if (dataLen == SQL_NO_TOTAL) { - LOG("SQLGetData couldn't determine the length of the data. " - "Returning NULL value instead. Column ID - {}, Data Type - {}", i, dataType); + LOG("SQLGetData: Cannot determine data length (SQL_NO_TOTAL) for column %d (SQL_CHAR), returning NULL", i); row.append(py::none()); } else if (dataLen < 0) { - LOG("SQLGetData returned an unexpected negative data length. " - "Raising exception. Column ID - {}, Data Type - {}, Data Length - {}", - i, dataType, dataLen); + LOG("SQLGetData: Unexpected negative data length for column %d - dataType=%d, dataLen=%ld", i, dataType, (long)dataLen); ThrowStdException("SQLGetData returned an unexpected negative data length"); } } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving data for column %d (SQL_CHAR) - SQLRETURN=%d, returning NULL", i, ret); row.append(py::none()); } } @@ -2538,7 +2673,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p } case SQL_SS_XML: { - LOG("Streaming XML for column {}", i); + LOG("SQLGetData: Streaming XML for column %d", i); row.append(FetchLobColumnData(hStmt, i, SQL_C_WCHAR, true, false)); break; } @@ -2546,7 +2681,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p case SQL_WVARCHAR: case SQL_WLONGVARCHAR: { if (columnSize == SQL_NO_TOTAL || columnSize > 4000) { - LOG("Streaming LOB for column {} (NVARCHAR)", i); + LOG("SQLGetData: Streaming LOB for column %d (SQL_C_WCHAR) - columnSize=%lu", i, (unsigned long)columnSize); row.append(FetchLobColumnData(hStmt, i, SQL_C_WCHAR, true, false)); } else { uint64_t fetchBufferSize = (columnSize + 1) * sizeof(SQLWCHAR); // +1 for null terminator @@ -2566,28 +2701,26 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p std::wstring wstr(reinterpret_cast(dataBuffer.data())); row.append(py::cast(wstr)); #endif - LOG("Appended NVARCHAR string of length {} to result row", numCharsInData); + LOG("SQLGetData: Appended NVARCHAR string length=%lu for column %d", (unsigned long)numCharsInData, i); } else { // Buffer too small, fallback to streaming - LOG("NVARCHAR column {} data truncated, using streaming LOB", i); + LOG("SQLGetData: NVARCHAR column %d data truncated, using streaming LOB", i); row.append(FetchLobColumnData(hStmt, i, SQL_C_WCHAR, true, false)); } } else if (dataLen == SQL_NULL_DATA) { - LOG("Column {} is NULL (CHAR)", i); + LOG("SQLGetData: Column %d is NULL (NVARCHAR)", i); row.append(py::none()); } else if (dataLen == 0) { row.append(py::str("")); } else if (dataLen == SQL_NO_TOTAL) { - LOG("SQLGetData couldn't determine the length of the NVARCHAR data. Returning NULL. Column ID - {}", i); + LOG("SQLGetData: Cannot determine NVARCHAR data length (SQL_NO_TOTAL) for column %d, returning NULL", i); row.append(py::none()); } else if (dataLen < 0) { - LOG("SQLGetData returned an unexpected negative data length. " - "Raising exception. Column ID - {}, Data Type - {}, Data Length - {}", - i, dataType, dataLen); + LOG("SQLGetData: Unexpected negative data length for column %d (NVARCHAR) - dataLen=%ld", i, (long)dataLen); ThrowStdException("SQLGetData returned an unexpected negative data length"); } } else { - LOG("Error retrieving data for column {} (NVARCHAR), SQLGetData return code {}", i, ret); + LOG("SQLGetData: Error retrieving data for column %d (NVARCHAR) - SQLRETURN=%d", i, ret); row.append(py::none()); } } @@ -2609,9 +2742,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p if (SQL_SUCCEEDED(ret)) { row.append(static_cast(smallIntValue)); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_SMALLINT for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } break; @@ -2622,9 +2753,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p if (SQL_SUCCEEDED(ret)) { row.append(realValue); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_REAL for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } break; @@ -2672,14 +2801,12 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p row.append(decimalObj); } catch (const py::error_already_set& e) { // If conversion fails, append None - LOG("Error converting to decimal: {}", e.what()); + LOG("SQLGetData: Error converting to decimal for column %d - %s", i, e.what()); row.append(py::none()); } } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_NUMERIC/DECIMAL for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } break; @@ -2692,9 +2819,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p if (SQL_SUCCEEDED(ret)) { row.append(doubleValue); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_DOUBLE/FLOAT for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } break; @@ -2705,9 +2830,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p if (SQL_SUCCEEDED(ret)) { row.append(static_cast(bigintValue)); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_BIGINT for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } break; @@ -2725,9 +2848,6 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p ) ); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); row.append(py::none()); } break; @@ -2747,9 +2867,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p ) ); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_TYPE_TIME for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } break; @@ -2773,9 +2891,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p ) ); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_TYPE_TIMESTAMP for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } break; @@ -2791,8 +2907,8 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p &indicator ); if (SQL_SUCCEEDED(ret) && indicator != SQL_NULL_DATA) { - LOG("[Fetch] Retrieved DTO: {}-{}-{} {}:{}:{}, fraction(ns)={}, tz_hour={}, tz_minute={}", - dtoValue.year, dtoValue.month, dtoValue.day, + LOG("SQLGetData: Retrieved DATETIMEOFFSET for column %d - %d-%d-%d %d:%d:%d, fraction_ns=%u, tz_hour=%d, tz_minute=%d", + i, dtoValue.year, dtoValue.month, dtoValue.day, dtoValue.hour, dtoValue.minute, dtoValue.second, dtoValue.fraction, dtoValue.timezone_hour, dtoValue.timezone_minute @@ -2824,7 +2940,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p ); row.append(py_dt); } else { - LOG("Error fetching DATETIMEOFFSET for column {}, ret={}", i, ret); + LOG("SQLGetData: Error fetching DATETIMEOFFSET for column %d - SQLRETURN=%d, indicator=%ld", i, ret, (long)indicator); row.append(py::none()); } break; @@ -2834,7 +2950,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p case SQL_LONGVARBINARY: { // Use streaming for large VARBINARY (columnSize unknown or > 8000) if (columnSize == SQL_NO_TOTAL || columnSize == 0 || columnSize > 8000) { - LOG("Streaming LOB for column {} (VARBINARY)", i); + LOG("SQLGetData: Streaming LOB for column %d (SQL_C_BINARY) - columnSize=%lu", i, (unsigned long)columnSize); row.append(FetchLobColumnData(hStmt, i, SQL_C_BINARY, false, true)); } else { // Small VARBINARY, fetch directly @@ -2847,7 +2963,6 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p if (static_cast(dataLen) <= columnSize) { row.append(py::bytes(reinterpret_cast(dataBuffer.data()), dataLen)); } else { - LOG("VARBINARY column {} data truncated, using streaming LOB", i); row.append(FetchLobColumnData(hStmt, i, SQL_C_BINARY, false, true)); } } else if (dataLen == SQL_NULL_DATA) { @@ -2858,11 +2973,11 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p std::ostringstream oss; oss << "Unexpected negative length (" << dataLen << ") returned by SQLGetData. ColumnID=" << i << ", dataType=" << dataType << ", bufferSize=" << columnSize; - LOG("Error: {}", oss.str()); + LOG("SQLGetData: %s", oss.str().c_str()); ThrowStdException(oss.str()); } } else { - LOG("Error retrieving VARBINARY data for column {}. SQLGetData rc = {}", i, ret); + LOG("SQLGetData: Error retrieving VARBINARY data for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } } @@ -2874,9 +2989,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p if (SQL_SUCCEEDED(ret)) { row.append(static_cast(tinyIntValue)); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_TINYINT for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } break; @@ -2887,9 +3000,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p if (SQL_SUCCEEDED(ret)) { row.append(static_cast(bitValue)); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_BIT for column %d - SQLRETURN=%d", i, ret); row.append(py::none()); } break; @@ -2919,9 +3030,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p } else if (indicator == SQL_NULL_DATA) { row.append(py::none()); } else { - LOG("Error retrieving data for column - {}, data type - {}, SQLGetData return " - "code - {}. Returning NULL value instead", - i, dataType, ret); + LOG("SQLGetData: Error retrieving SQL_GUID for column %d - SQLRETURN=%d, indicator=%ld", i, ret, (long)indicator); row.append(py::none()); } break; @@ -2931,7 +3040,7 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p std::ostringstream errorString; errorString << "Unsupported data type for column - " << columnName << ", Type - " << dataType << ", column ID - " << i; - LOG(errorString.str()); + LOG("SQLGetData: %s", errorString.str().c_str()); ThrowStdException(errorString.str()); break; } @@ -2940,9 +3049,9 @@ SQLRETURN SQLGetData_wrap(SqlHandlePtr StatementHandle, SQLUSMALLINT colCount, p } SQLRETURN SQLFetchScroll_wrap(SqlHandlePtr StatementHandle, SQLSMALLINT FetchOrientation, SQLLEN FetchOffset, py::list& row_data) { - LOG("Fetching with scroll: orientation={}, offset={}", FetchOrientation, FetchOffset); + LOG("SQLFetchScroll_wrap: Fetching with scroll orientation=%d, offset=%ld", FetchOrientation, (long)FetchOffset); if (!SQLFetchScroll_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLFetchScroll_wrap: Function pointer not initialized. Loading the driver."); DriverLoader::getInstance().loadDriver(); // Load the driver } @@ -3104,7 +3213,7 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column std::ostringstream errorString; errorString << "Unsupported data type for column - " << columnName.c_str() << ", Type - " << dataType << ", column ID - " << col; - LOG(errorString.str()); + LOG("SQLBindColums: %s", errorString.str().c_str()); ThrowStdException(errorString.str()); break; } @@ -3113,7 +3222,7 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column std::ostringstream errorString; errorString << "Failed to bind column - " << columnName.c_str() << ", Type - " << dataType << ", column ID - " << col; - LOG(errorString.str()); + LOG("SQLBindColums: %s", errorString.str().c_str()); ThrowStdException(errorString.str()); return ret; } @@ -3125,14 +3234,14 @@ SQLRETURN SQLBindColums(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& column // TODO: Move to anonymous namespace, since it is not used outside this file SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& columnNames, py::list& rows, SQLUSMALLINT numCols, SQLULEN& numRowsFetched, const std::vector& lobColumns) { - LOG("Fetching data in batches"); + LOG("FetchBatchData: Fetching data in batches"); SQLRETURN ret = SQLFetchScroll_ptr(hStmt, SQL_FETCH_NEXT, 0); if (ret == SQL_NO_DATA) { - LOG("No data to fetch"); + LOG("FetchBatchData: No data to fetch"); return ret; } if (!SQL_SUCCEEDED(ret)) { - LOG("Error while fetching rows in batches"); + LOG("FetchBatchData: Error while fetching rows in batches - SQLRETURN=%d", ret); return ret; } // numRowsFetched is the SQL_ATTR_ROWS_FETCHED_PTR attribute. It'll be populated by @@ -3151,12 +3260,11 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum // TODO: variable length data needs special handling, this logic wont suffice // This value indicates that the driver cannot determine the length of the data if (dataLen == SQL_NO_TOTAL) { - LOG("Cannot determine the length of the data. Returning NULL value instead." - "Column ID - {}", col); + LOG("FetchBatchData: Cannot determine data length for column %d - returning NULL", col); row.append(py::none()); continue; } else if (dataLen == SQL_NULL_DATA) { - LOG("Column data is NULL. Appending None to the result row. Column ID - {}", col); + LOG("FetchBatchData: Column %d data is NULL", col); row.append(py::none()); continue; } else if (dataLen == 0) { @@ -3169,13 +3277,13 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum row.append(py::bytes("")); } else { // For other datatypes, 0 length is unexpected. Log & append None - LOG("Column data length is 0 for non-string/binary datatype. Appending None to the result row. Column ID - {}", col); + LOG("FetchBatchData: Unexpected 0-length data for column %d (type=%d) - returning NULL", col, dataType); row.append(py::none()); } continue; } else if (dataLen < 0) { // Negative value is unexpected, log column index, SQL type & raise exception - LOG("Unexpected negative data length. Column ID - {}, SQL Type - {}, Data Length - {}", col, dataType, dataLen); + LOG("FetchBatchData: Unexpected negative data length - column=%d, SQL_type=%d, dataLen=%ld", col, dataType, (long)dataLen); ThrowStdException("Unexpected negative data length, check logs for details"); } assert(dataLen > 0 && "Data length must be > 0"); @@ -3271,7 +3379,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum row.append(py::module_::import("decimal").attr("Decimal")(numStr)); } catch (const py::error_already_set& e) { // Handle the exception, e.g., log the error and append py::none() - LOG("Error converting to decimal: {}", e.what()); + LOG("FetchAll_wrap: Error converting to decimal - %s", e.what()); row.append(py::none()); } break; @@ -3385,7 +3493,7 @@ SQLRETURN FetchBatchData(SQLHSTMT hStmt, ColumnBuffers& buffers, py::list& colum std::ostringstream errorString; errorString << "Unsupported data type for column - " << columnName.c_str() << ", Type - " << dataType << ", column ID - " << col; - LOG(errorString.str()); + LOG("FetchBatchData: %s", errorString.str().c_str()); ThrowStdException(errorString.str()); break; } @@ -3473,7 +3581,7 @@ size_t calculateRowSize(py::list& columnNames, SQLUSMALLINT numCols) { std::ostringstream errorString; errorString << "Unsupported data type for column - " << columnName.c_str() << ", Type - " << dataType << ", column ID - " << col; - LOG(errorString.str()); + LOG("calculateRowSize: %s", errorString.str().c_str()); ThrowStdException(errorString.str()); break; } @@ -3505,7 +3613,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch py::list columnNames; ret = SQLDescribeCol_wrap(StatementHandle, columnNames); if (!SQL_SUCCEEDED(ret)) { - LOG("Failed to get column descriptions"); + LOG("FetchMany_wrap: Failed to get column descriptions - SQLRETURN=%d", ret); return ret; } @@ -3525,7 +3633,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch // If we have LOBs → fall back to row-by-row fetch + SQLGetData_wrap if (!lobColumns.empty()) { - LOG("LOB columns detected, using per-row SQLGetData path"); + LOG("FetchMany_wrap: LOB columns detected (%zu columns), using per-row SQLGetData path", lobColumns.size()); while (true) { ret = SQLFetch_ptr(hStmt); if (ret == SQL_NO_DATA) break; @@ -3544,7 +3652,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch // Bind columns ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize); if (!SQL_SUCCEEDED(ret)) { - LOG("Error when binding columns"); + LOG("FetchMany_wrap: Error when binding columns - SQLRETURN=%d", ret); return ret; } @@ -3554,7 +3662,7 @@ SQLRETURN FetchMany_wrap(SqlHandlePtr StatementHandle, py::list& rows, int fetch ret = FetchBatchData(hStmt, buffers, columnNames, rows, numCols, numRowsFetched, lobColumns); if (!SQL_SUCCEEDED(ret) && ret != SQL_NO_DATA) { - LOG("Error when fetching data"); + LOG("FetchMany_wrap: Error when fetching data - SQLRETURN=%d", ret); return ret; } @@ -3588,7 +3696,7 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows) { py::list columnNames; ret = SQLDescribeCol_wrap(StatementHandle, columnNames); if (!SQL_SUCCEEDED(ret)) { - LOG("Failed to get column descriptions"); + LOG("FetchAll_wrap: Failed to get column descriptions - SQLRETURN=%d", ret); return ret; } @@ -3629,7 +3737,7 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows) { } else { fetchSize = 1000; } - LOG("Fetching data in batch sizes of {}", fetchSize); + LOG("FetchAll_wrap: Fetching data in batch sizes of %d", fetchSize); std::vector lobColumns; for (SQLSMALLINT i = 0; i < numCols; i++) { @@ -3647,7 +3755,7 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows) { // If we have LOBs → fall back to row-by-row fetch + SQLGetData_wrap if (!lobColumns.empty()) { - LOG("LOB columns detected, using per-row SQLGetData path"); + LOG("FetchAll_wrap: LOB columns detected (%zu columns), using per-row SQLGetData path", lobColumns.size()); while (true) { ret = SQLFetch_ptr(hStmt); if (ret == SQL_NO_DATA) break; @@ -3665,7 +3773,7 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows) { // Bind columns ret = SQLBindColums(hStmt, buffers, columnNames, numCols, fetchSize); if (!SQL_SUCCEEDED(ret)) { - LOG("Error when binding columns"); + LOG("FetchAll_wrap: Error when binding columns - SQLRETURN=%d", ret); return ret; } @@ -3676,7 +3784,7 @@ SQLRETURN FetchAll_wrap(SqlHandlePtr StatementHandle, py::list& rows) { while (ret != SQL_NO_DATA) { ret = FetchBatchData(hStmt, buffers, columnNames, rows, numCols, numRowsFetched, lobColumns); if (!SQL_SUCCEEDED(ret) && ret != SQL_NO_DATA) { - LOG("Error when fetching data"); + LOG("FetchAll_wrap: Error when fetching data - SQLRETURN=%d", ret); return ret; } } @@ -3712,16 +3820,16 @@ SQLRETURN FetchOne_wrap(SqlHandlePtr StatementHandle, py::list& row) { SQLSMALLINT colCount = SQLNumResultCols_wrap(StatementHandle); ret = SQLGetData_wrap(StatementHandle, colCount, row); } else if (ret != SQL_NO_DATA) { - LOG("Error when fetching data"); + LOG("FetchOne_wrap: Error when fetching data - SQLRETURN=%d", ret); } return ret; } // Wrap SQLMoreResults SQLRETURN SQLMoreResults_wrap(SqlHandlePtr StatementHandle) { - LOG("Check for more results"); + LOG("SQLMoreResults_wrap: Check for more results"); if (!SQLMoreResults_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLMoreResults_wrap: Function pointer not initialized. Loading the driver."); DriverLoader::getInstance().loadDriver(); // Load the driver } @@ -3730,15 +3838,15 @@ SQLRETURN SQLMoreResults_wrap(SqlHandlePtr StatementHandle) { // Wrap SQLFreeHandle SQLRETURN SQLFreeHandle_wrap(SQLSMALLINT HandleType, SqlHandlePtr Handle) { - LOG("Free SQL handle"); + LOG("SQLFreeHandle_wrap: Free SQL handle type=%d", HandleType); if (!SQLAllocHandle_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLFreeHandle_wrap: Function pointer not initialized. Loading the driver."); DriverLoader::getInstance().loadDriver(); // Load the driver } SQLRETURN ret = SQLFreeHandle_ptr(HandleType, Handle->get()); if (!SQL_SUCCEEDED(ret)) { - LOG("SQLFreeHandle failed with error code - {}", ret); + LOG("SQLFreeHandle_wrap: SQLFreeHandle failed with error code - %d", ret); return ret; } return ret; @@ -3746,19 +3854,19 @@ SQLRETURN SQLFreeHandle_wrap(SQLSMALLINT HandleType, SqlHandlePtr Handle) { // Wrap SQLRowCount SQLLEN SQLRowCount_wrap(SqlHandlePtr StatementHandle) { - LOG("Get number of row affected by last execute"); + LOG("SQLRowCount_wrap: Get number of rows affected by last execute"); if (!SQLRowCount_ptr) { - LOG("Function pointer not initialized. Loading the driver."); + LOG("SQLRowCount_wrap: Function pointer not initialized. Loading the driver."); DriverLoader::getInstance().loadDriver(); // Load the driver } SQLLEN rowCount; SQLRETURN ret = SQLRowCount_ptr(StatementHandle->get(), &rowCount); if (!SQL_SUCCEEDED(ret)) { - LOG("SQLRowCount failed with error code - {}", ret); + LOG("SQLRowCount_wrap: SQLRowCount failed with error code - %d", ret); return ret; } - LOG("SQLRowCount returned {}", rowCount); + LOG("SQLRowCount_wrap: SQLRowCount returned %ld", (long)rowCount); return rowCount; } @@ -3933,12 +4041,24 @@ PYBIND11_MODULE(ddbc_bindings, m) { // Add a version attribute m.attr("__version__") = "1.0.0"; + // Expose logger bridge function to Python + m.def("update_log_level", &mssql_python::logging::LoggerBridge::updateLevel, + "Update the cached log level in C++ bridge"); + + // Initialize the logger bridge + try { + mssql_python::logging::LoggerBridge::initialize(); + } catch (const std::exception& e) { + // Log initialization failure but don't throw + fprintf(stderr, "Logger bridge initialization failed: %s\n", e.what()); + } + try { // Try loading the ODBC driver when the module is imported - LOG("Loading ODBC driver"); + LOG("Module initialization: Loading ODBC driver"); DriverLoader::getInstance().loadDriver(); // Load the driver } catch (const std::exception& e) { // Log the error but don't throw - let the error happen when functions are called - LOG("Failed to load ODBC driver during module initialization: {}", e.what()); + LOG("Module initialization: Failed to load ODBC driver - %s", e.what()); } } diff --git a/mssql_python/pybind/ddbc_bindings.h b/mssql_python/pybind/ddbc_bindings.h index eeb5bb37..03fd21e8 100644 --- a/mssql_python/pybind/ddbc_bindings.h +++ b/mssql_python/pybind/ddbc_bindings.h @@ -32,6 +32,9 @@ using py::literals::operator""_a; #include #include +// Include logger bridge for LOG macros +#include "logger_bridge.hpp" + #if defined(_WIN32) inline std::vector WStringToSQLWCHAR(const std::wstring& str) { std::vector result(str.begin(), str.end()); @@ -362,10 +365,6 @@ extern SQLDescribeParamFunc SQLDescribeParam_ptr; extern SQLParamDataFunc SQLParamData_ptr; extern SQLPutDataFunc SQLPutData_ptr; -// Logging utility -template -void LOG(const std::string& formatString, Args&&... args); - // Throws a std::runtime_error with the given message void ThrowStdException(const std::string& message); @@ -505,7 +504,7 @@ inline std::wstring Utf8ToWString(const std::string& str) { static_cast(str.size()), nullptr, 0); if (size_needed == 0) { - LOG("MultiByteToWideChar failed."); + LOG_ERROR("MultiByteToWideChar failed for UTF8 to wide string conversion"); return {}; } std::wstring result(size_needed, 0); diff --git a/mssql_python/pybind/logger_bridge.cpp b/mssql_python/pybind/logger_bridge.cpp new file mode 100644 index 00000000..a28c10df --- /dev/null +++ b/mssql_python/pybind/logger_bridge.cpp @@ -0,0 +1,208 @@ +/** + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + * + * Logger Bridge Implementation + */ + +#include "logger_bridge.hpp" +#include +#include +#include +#include +#include + +namespace mssql_python { +namespace logging { + +// Initialize static members +PyObject* LoggerBridge::cached_logger_ = nullptr; +std::atomic LoggerBridge::cached_level_(LOG_LEVEL_CRITICAL); // Disabled by default +std::mutex LoggerBridge::mutex_; +bool LoggerBridge::initialized_ = false; + +void LoggerBridge::initialize() { + std::lock_guard lock(mutex_); + + // Skip if already initialized + if (initialized_) { + return; + } + + try { + // Acquire GIL for Python API calls + py::gil_scoped_acquire gil; + + // Import the logging module + py::module_ logging_module = py::module_::import("mssql_python.logging"); + + // Get the logger instance + py::object logger_obj = logging_module.attr("logger"); + + // Cache the logger object (increment refcount) + cached_logger_ = logger_obj.ptr(); + Py_INCREF(cached_logger_); + + // Get initial log level + py::object level_obj = logger_obj.attr("level"); + int level = level_obj.cast(); + cached_level_.store(level, std::memory_order_relaxed); + + initialized_ = true; + + } catch (const py::error_already_set& e) { + // Failed to initialize - log to stderr and continue + // (logging will be disabled but won't crash) + std::cerr << "LoggerBridge initialization failed: " << e.what() << std::endl; + initialized_ = false; + } catch (const std::exception& e) { + std::cerr << "LoggerBridge initialization failed: " << e.what() << std::endl; + initialized_ = false; + } +} + +void LoggerBridge::updateLevel(int level) { + // Update the cached level atomically + // This is lock-free and can be called from any thread + cached_level_.store(level, std::memory_order_relaxed); +} + +int LoggerBridge::getLevel() { + return cached_level_.load(std::memory_order_relaxed); +} + +bool LoggerBridge::isInitialized() { + return initialized_; +} + +std::string LoggerBridge::formatMessage(const char* format, va_list args) { + // Use a stack buffer for most messages (4KB should be enough) + char buffer[4096]; + + // Format the message using safe std::vsnprintf (C++11 standard) + // std::vsnprintf is safe: always null-terminates, never overflows buffer + // DevSkim warning is false positive - this is the recommended safe alternative + va_list args_copy; + va_copy(args_copy, args); + int result = std::vsnprintf(buffer, sizeof(buffer), format, args_copy); + va_end(args_copy); + + if (result < 0) { + // Error during formatting + return "[Formatting error]"; + } + + if (result < static_cast(sizeof(buffer))) { + // Message fit in buffer (vsnprintf guarantees null-termination) + return std::string(buffer, std::min(static_cast(result), sizeof(buffer) - 1)); + } + + // Message was truncated - allocate larger buffer + // (This should be rare for typical log messages) + std::vector large_buffer(result + 1); + va_copy(args_copy, args); + // std::vsnprintf is safe here too - proper bounds checking with buffer size + std::vsnprintf(large_buffer.data(), large_buffer.size(), format, args_copy); + va_end(args_copy); + + return std::string(large_buffer.data()); +} + +const char* LoggerBridge::extractFilename(const char* path) { + // Extract just the filename from full path using safer C++ string search + if (!path) { + return ""; + } + + // Find last occurrence of Unix path separator + const char* filename = std::strrchr(path, '/'); + if (filename) { + return filename + 1; + } + + // Try Windows path separator + filename = std::strrchr(path, '\\'); + if (filename) { + return filename + 1; + } + + // No path separator found, return the whole string + return path; +} + +void LoggerBridge::log(int level, const char* file, int line, + const char* format, ...) { + // Fast level check (should already be done by macro, but double-check) + if (!isLoggable(level)) { + return; + } + + // Check if initialized + if (!initialized_ || !cached_logger_) { + return; + } + + // Format the message + va_list args; + va_start(args, format); + std::string message = formatMessage(format, args); + va_end(args); + + // Extract filename from path + const char* filename = extractFilename(file); + + // Format the complete log message with [DDBC] prefix for CSV parsing + // File and line number are handled by the Python formatter (in Location column) + // std::snprintf is safe: always null-terminates, never overflows buffer + // DevSkim warning is false positive - this is the recommended safe alternative + char complete_message[4096]; + int written = std::snprintf(complete_message, sizeof(complete_message), + "[DDBC] %s", message.c_str()); + + // Ensure null-termination (snprintf guarantees this, but be explicit) + if (written >= static_cast(sizeof(complete_message))) { + complete_message[sizeof(complete_message) - 1] = '\0'; + } + + // Lock for Python call (minimize critical section) + std::lock_guard lock(mutex_); + + try { + // Acquire GIL for Python API call + py::gil_scoped_acquire gil; + + // Get the logger object + py::handle logger_handle(cached_logger_); + py::object logger_obj = py::reinterpret_borrow(logger_handle); + + // Get the underlying Python logger to create LogRecord with correct filename/lineno + py::object py_logger = logger_obj.attr("_logger"); + + // Call makeRecord to create a LogRecord with correct attributes + py::object record = py_logger.attr("makeRecord")( + py_logger.attr("name"), // name + py::int_(level), // level + py::str(filename), // pathname (just filename) + py::int_(line), // lineno + py::str(complete_message), // msg + py::tuple(), // args + py::none(), // exc_info + py::str(filename), // func (use filename as func name) + py::none() // extra + ); + + // Call handle() to process the record through filters and handlers + py_logger.attr("handle")(record); + + } catch (const py::error_already_set& e) { + // Python error during logging - ignore to prevent cascading failures + // (Logging errors should not crash the application) + (void)e; // Suppress unused variable warning + } catch (const std::exception& e) { + // Other error - ignore + (void)e; + } +} + +} // namespace logging +} // namespace mssql_python diff --git a/mssql_python/pybind/logger_bridge.hpp b/mssql_python/pybind/logger_bridge.hpp new file mode 100644 index 00000000..a4e6683f --- /dev/null +++ b/mssql_python/pybind/logger_bridge.hpp @@ -0,0 +1,177 @@ +/** + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT license. + * + * Logger Bridge for mssql_python - High-performance logging from C++ to Python + * + * This bridge provides zero-overhead logging when disabled via: + * - Cached Python logger object (import once) + * - Atomic log level storage (lock-free reads) + * - Fast inline level checks + * - Lazy message formatting + */ + +#ifndef MSSQL_PYTHON_LOGGER_BRIDGE_HPP +#define MSSQL_PYTHON_LOGGER_BRIDGE_HPP + +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace mssql_python { +namespace logging { + +// Log level constants (matching Python levels) +// Note: Avoid using ERROR as it conflicts with Windows.h macro +const int LOG_LEVEL_DEBUG = 10; // Debug/diagnostic logging +const int LOG_LEVEL_INFO = 20; // Informational +const int LOG_LEVEL_WARNING = 30; // Warnings +const int LOG_LEVEL_ERROR = 40; // Errors +const int LOG_LEVEL_CRITICAL = 50; // Critical errors + +/** + * LoggerBridge - Bridge between C++ and Python logging + * + * Features: + * - Singleton pattern + * - Cached Python logger (imported once) + * - Atomic level check (zero overhead) + * - Thread-safe + * - GIL-aware + */ +class LoggerBridge { +public: + /** + * Initialize the logger bridge. + * Must be called once during module initialization. + * Caches the Python logger object and initial level. + */ + static void initialize(); + + /** + * Update the cached log level. + * Called from Python when logger.setLevel() is invoked. + * + * @param level New log level + */ + static void updateLevel(int level); + + /** + * Fast check if a log level is enabled. + * This is inline and lock-free for zero overhead. + * + * @param level Log level to check + * @return true if level is enabled, false otherwise + */ + static inline bool isLoggable(int level) { + return level >= cached_level_.load(std::memory_order_relaxed); + } + + /** + * Log a message at the specified level. + * Only call this if isLoggable() returns true. + * + * @param level Log level + * @param file Source file name (__FILE__) + * @param line Line number (__LINE__) + * @param format Printf-style format string + * @param ... Variable arguments for format string + */ + static void log(int level, const char* file, int line, + const char* format, ...); + + /** + * Get the current log level. + * + * @return Current log level + */ + static int getLevel(); + + /** + * Check if the bridge is initialized. + * + * @return true if initialized, false otherwise + */ + static bool isInitialized(); + +private: + // Private constructor (singleton) + LoggerBridge() = default; + + // No copying or moving + LoggerBridge(const LoggerBridge&) = delete; + LoggerBridge& operator=(const LoggerBridge&) = delete; + + // Cached Python logger object + static PyObject* cached_logger_; + + // Cached log level (atomic for lock-free reads) + static std::atomic cached_level_; + + // Mutex for initialization and Python calls + static std::mutex mutex_; + + // Initialization flag + static bool initialized_; + + /** + * Helper to format message with va_list. + * + * @param format Printf-style format string + * @param args Variable arguments + * @return Formatted string + */ + static std::string formatMessage(const char* format, va_list args); + + /** + * Helper to extract filename from full path. + * + * @param path Full file path + * @return Filename only + */ + static const char* extractFilename(const char* path); +}; + +} // namespace logging +} // namespace mssql_python + +// Convenience macros for logging +// Single LOG() macro for all diagnostic logging (DEBUG level) + +#define LOG(fmt, ...) \ + do { \ + if (mssql_python::logging::LoggerBridge::isLoggable(mssql_python::logging::LOG_LEVEL_DEBUG)) { \ + mssql_python::logging::LoggerBridge::log( \ + mssql_python::logging::LOG_LEVEL_DEBUG, __FILE__, __LINE__, fmt, ##__VA_ARGS__); \ + } \ + } while(0) + +#define LOG_INFO(fmt, ...) \ + do { \ + if (mssql_python::logging::LoggerBridge::isLoggable(mssql_python::logging::LOG_LEVEL_INFO)) { \ + mssql_python::logging::LoggerBridge::log( \ + mssql_python::logging::LOG_LEVEL_INFO, __FILE__, __LINE__, fmt, ##__VA_ARGS__); \ + } \ + } while(0) + +#define LOG_WARNING(fmt, ...) \ + do { \ + if (mssql_python::logging::LoggerBridge::isLoggable(mssql_python::logging::LOG_LEVEL_WARNING)) { \ + mssql_python::logging::LoggerBridge::log( \ + mssql_python::logging::LOG_LEVEL_WARNING, __FILE__, __LINE__, fmt, ##__VA_ARGS__); \ + } \ + } while(0) + +#define LOG_ERROR(fmt, ...) \ + do { \ + if (mssql_python::logging::LoggerBridge::isLoggable(mssql_python::logging::LOG_LEVEL_ERROR)) { \ + mssql_python::logging::LoggerBridge::log( \ + mssql_python::logging::LOG_LEVEL_ERROR, __FILE__, __LINE__, fmt, ##__VA_ARGS__); \ + } \ + } while(0) + +#endif // MSSQL_PYTHON_LOGGER_BRIDGE_HPP diff --git a/mssql_python/pybind/unix_utils.cpp b/mssql_python/pybind/unix_utils.cpp index 3fd325bd..d8630b36 100644 --- a/mssql_python/pybind/unix_utils.cpp +++ b/mssql_python/pybind/unix_utils.cpp @@ -7,6 +7,7 @@ // differences specific to macOS. #include "unix_utils.h" +#include "logger_bridge.hpp" #include #include #include @@ -17,39 +18,25 @@ const char* kOdbcEncoding = "utf-16-le"; // ODBC uses UTF-16LE for SQLWCHAR const size_t kUcsLength = 2; // SQLWCHAR is 2 bytes on all platforms -// TODO(microsoft): Make Logger a separate module and import it across project -template -void LOG(const std::string& formatString, Args&&... args) { - py::gil_scoped_acquire gil; // this ensures safe Python API usage - - py::object logger = py::module_::import("mssql_python.logging_config") - .attr("get_logger")(); - if (py::isinstance(logger)) return; - - try { - std::string ddbcFormatString = "[DDBC Bindings log] " + formatString; - if constexpr (sizeof...(args) == 0) { - logger.attr("debug")(py::str(ddbcFormatString)); - } else { - py::str message = py::str(ddbcFormatString) - .format(std::forward(args)...); - logger.attr("debug")(message); - } - } catch (const std::exception& e) { - std::cerr << "Logging error: " << e.what() << std::endl; - } -} +// Logging uses LOG() macro for all diagnostic output +#define LOG(...) do {} while(0) // Function to convert SQLWCHAR strings to std::wstring on macOS std::wstring SQLWCHARToWString(const SQLWCHAR* sqlwStr, size_t length = SQL_NTS) { - if (!sqlwStr) return std::wstring(); + if (!sqlwStr) { + LOG("SQLWCHARToWString: NULL input - returning empty wstring"); + return std::wstring(); + } if (length == SQL_NTS) { // Determine length if not provided size_t i = 0; while (sqlwStr[i] != 0) ++i; length = i; + LOG("SQLWCHARToWString: Length determined - length=%zu", length); + } else { + LOG("SQLWCHARToWString: Using provided length=%zu", length); } // Create a UTF-16LE byte array from the SQLWCHAR array @@ -58,6 +45,7 @@ std::wstring SQLWCHARToWString(const SQLWCHAR* sqlwStr, // Copy each SQLWCHAR (2 bytes) to the byte array memcpy(&utf16Bytes[i * kUcsLength], &sqlwStr[i], kUcsLength); } + LOG("SQLWCHARToWString: UTF-16LE byte array created - byte_count=%zu", utf16Bytes.size()); // Convert UTF-16LE to std::wstring (UTF-32 on macOS) try { @@ -65,32 +53,36 @@ std::wstring SQLWCHARToWString(const SQLWCHAR* sqlwStr, std::wstring_convert> converter; - return converter.from_bytes( + std::wstring result = converter.from_bytes( reinterpret_cast(utf16Bytes.data()), reinterpret_cast(utf16Bytes.data() + utf16Bytes.size())); + LOG("SQLWCHARToWString: Conversion successful - input_len=%zu, result_len=%zu", + length, result.size()); + return result; } catch (const std::exception& e) { - // Log a warning about using fallback conversion - LOG("Warning: Using fallback string conversion on macOS. " - "Character data might be inexact."); // Fallback to character-by-character conversion if codecvt fails + LOG("SQLWCHARToWString: codecvt failed (%s), using fallback - length=%zu", e.what(), length); std::wstring result; result.reserve(length); for (size_t i = 0; i < length; ++i) { result.push_back(static_cast(sqlwStr[i])); } + LOG("SQLWCHARToWString: Fallback conversion complete - result_len=%zu", result.size()); return result; } } // Function to convert std::wstring to SQLWCHAR array on macOS std::vector WStringToSQLWCHAR(const std::wstring& str) { + LOG("WStringToSQLWCHAR: Starting conversion - input_len=%zu", str.size()); try { // Convert wstring (UTF-32 on macOS) to UTF-16LE bytes std::wstring_convert> converter; std::string utf16Bytes = converter.to_bytes(str); + LOG("WStringToSQLWCHAR: UTF-16LE byte conversion successful - byte_count=%zu", utf16Bytes.size()); // Convert the bytes to SQLWCHAR array std::vector result(utf16Bytes.size() / kUcsLength + 1, @@ -98,17 +90,17 @@ std::vector WStringToSQLWCHAR(const std::wstring& str) { for (size_t i = 0; i < utf16Bytes.size() / kUcsLength; ++i) { memcpy(&result[i], &utf16Bytes[i * kUcsLength], kUcsLength); } + LOG("WStringToSQLWCHAR: Conversion complete - result_size=%zu (includes null terminator)", result.size()); return result; } catch (const std::exception& e) { - // Log a warning about using fallback conversion - LOG("Warning: Using fallback conversion for std::wstring to " - "SQLWCHAR on macOS. Character data might be inexact."); // Fallback to simple casting if codecvt fails + LOG("WStringToSQLWCHAR: codecvt failed (%s), using fallback - input_len=%zu", e.what(), str.size()); std::vector result(str.size() + 1, 0); // +1 for null terminator for (size_t i = 0; i < str.size(); ++i) { result[i] = static_cast(str[i]); } + LOG("WStringToSQLWCHAR: Fallback conversion complete - result_size=%zu", result.size()); return result; } } @@ -116,7 +108,10 @@ std::vector WStringToSQLWCHAR(const std::wstring& str) { // This function can be used as a safe decoder for SQLWCHAR buffers // based on your ctypes UCS_dec implementation std::string SQLWCHARToUTF8String(const SQLWCHAR* buffer) { - if (!buffer) return ""; + if (!buffer) { + LOG("SQLWCHARToUTF8String: NULL buffer - returning empty string"); + return ""; + } std::vector utf16Bytes; size_t i = 0; @@ -127,28 +122,34 @@ std::string SQLWCHARToUTF8String(const SQLWCHAR* buffer) { utf16Bytes.push_back(bytes[1]); i++; } + LOG("SQLWCHARToUTF8String: UTF-16 bytes collected - char_count=%zu, byte_count=%zu", i, utf16Bytes.size()); try { std::wstring_convert> converter; - return converter.to_bytes( + std::string result = converter.to_bytes( reinterpret_cast(utf16Bytes.data()), reinterpret_cast(utf16Bytes.data() + utf16Bytes.size())); + LOG("SQLWCHARToUTF8String: UTF-8 conversion successful - input_chars=%zu, output_bytes=%zu", + i, result.size()); + return result; } catch (const std::exception& e) { - // Log a warning about using fallback conversion - LOG("Warning: Using fallback conversion for SQLWCHAR to UTF-8 " - "on macOS. Character data might be inexact."); // Simple fallback conversion + LOG("SQLWCHARToUTF8String: codecvt failed (%s), using ASCII fallback - char_count=%zu", e.what(), i); std::string result; + size_t non_ascii_count = 0; for (size_t j = 0; j < i; ++j) { if (buffer[j] < 128) { result.push_back(static_cast(buffer[j])); } else { result.push_back('?'); // Placeholder for non-ASCII chars + non_ascii_count++; } } + LOG("SQLWCHARToUTF8String: Fallback complete - output_bytes=%zu, non_ascii_replaced=%zu", + result.size(), non_ascii_count); return result; } } @@ -157,11 +158,14 @@ std::string SQLWCHARToUTF8String(const SQLWCHAR* buffer) { // This will process WCHAR data safely in SQLWCHARToUTF8String void SafeProcessWCharData(SQLWCHAR* buffer, SQLLEN indicator, py::list& row) { if (indicator == SQL_NULL_DATA) { + LOG("SafeProcessWCharData: NULL data - appending None"); row.append(py::none()); } else { // Use our safe conversion function + LOG("SafeProcessWCharData: Converting WCHAR data - indicator=%lld", static_cast(indicator)); std::string str = SQLWCHARToUTF8String(buffer); row.append(py::str(str)); + LOG("SafeProcessWCharData: String appended - length=%zu", str.size()); } } #endif diff --git a/mssql_python/row.py b/mssql_python/row.py index 8ffcb6e0..778f32c3 100644 --- a/mssql_python/row.py +++ b/mssql_python/row.py @@ -8,6 +8,7 @@ import uuid from typing import Any, Dict, List, Optional, Tuple, TYPE_CHECKING +from mssql_python.logging import logger from mssql_python.constants import ConstantsDDBC from mssql_python.helpers import get_settings @@ -105,22 +106,31 @@ def _process_uuid_values( Convert string UUIDs to uuid.UUID objects if native_uuid setting is True, or ensure UUIDs are returned as strings if False. """ + from mssql_python.logging import logger # Use the snapshot setting for native_uuid native_uuid = self._settings.get("native_uuid") + logger.debug( '_process_uuid_values: Processing - native_uuid=%s, value_count=%d', + str(native_uuid), len(values)) # Early return if no conversion needed - if not native_uuid and not any(isinstance(v, uuid.UUID) for v in values): + uuid_count = sum(1 for v in values if isinstance(v, uuid.UUID)) + if not native_uuid and uuid_count == 0: + logger.debug( '_process_uuid_values: No conversion needed - early return') return values # Get pre-identified UUID indices from cursor if available uuid_indices = getattr(self._cursor, "_uuid_indices", None) processed_values = list(values) # Create a copy to modify + logger.debug( '_process_uuid_values: uuid_indices=%s', + str(uuid_indices) if uuid_indices else 'None (will scan)') # Process only UUID columns when native_uuid is True if native_uuid: + conversion_count = 0 # If we have pre-identified UUID columns if uuid_indices is not None: + logger.debug( '_process_uuid_values: Using pre-identified indices - count=%d', len(uuid_indices)) for i in uuid_indices: if i < len(processed_values) and processed_values[i] is not None: value = processed_values[i] @@ -129,10 +139,14 @@ def _process_uuid_values( # Remove braces if present clean_value = value.strip("{}") processed_values[i] = uuid.UUID(clean_value) + conversion_count += 1 except (ValueError, AttributeError): + logger.debug( '_process_uuid_values: Conversion failed for index=%d', i) pass # Keep original if conversion fails + logger.debug( '_process_uuid_values: Converted %d UUID strings to UUID objects', conversion_count) # Fallback to scanning all columns if indices weren't pre-identified else: + logger.debug( '_process_uuid_values: Scanning all columns for GUID type') for i, value in enumerate(processed_values): if value is None: continue @@ -144,13 +158,19 @@ def _process_uuid_values( if isinstance(value, str): try: processed_values[i] = uuid.UUID(value.strip("{}")) + conversion_count += 1 except (ValueError, AttributeError): + logger.debug( '_process_uuid_values: Scan conversion failed for index=%d', i) pass + logger.debug( '_process_uuid_values: Scan converted %d UUID strings', conversion_count) # When native_uuid is False, convert UUID objects to strings else: + string_conversion_count = 0 for i, value in enumerate(processed_values): if isinstance(value, uuid.UUID): processed_values[i] = str(value) + string_conversion_count += 1 + logger.debug( '_process_uuid_values: Converted %d UUID objects to strings', string_conversion_count) return processed_values @@ -164,9 +184,14 @@ def _apply_output_converters(self, values: List[Any]) -> List[Any]: Returns: List of converted values """ + from mssql_python.logging import logger + if not self._description: + logger.debug( '_apply_output_converters: No description - returning values as-is') return values + logger.debug( '_apply_output_converters: Applying converters - value_count=%d', len(values)) + converted_values = list(values) # Map SQL type codes to appropriate byte sizes diff --git a/mssql_python/type.py b/mssql_python/type.py index 570d378d..6a68014e 100644 --- a/mssql_python/type.py +++ b/mssql_python/type.py @@ -7,6 +7,8 @@ import datetime import time +from mssql_python.logging import logger + # Type Objects class STRING(str): diff --git a/tests/test_004_cursor.py b/tests/test_004_cursor.py index b52b0656..0974eda7 100644 --- a/tests/test_004_cursor.py +++ b/tests/test_004_cursor.py @@ -3472,7 +3472,7 @@ def test_cursor_rownumber_empty_results(cursor, db_connection): def test_rownumber_warning_logged(cursor, db_connection): """Test that accessing rownumber logs a warning message""" import logging - from mssql_python.helpers import get_logger + from mssql_python.logging import get_logger try: # Create test table @@ -3487,6 +3487,12 @@ def test_rownumber_warning_logged(cursor, db_connection): # Set up logging capture logger = get_logger() if logger: + # Save original log level + original_level = logger._logger.level + + # Enable WARNING level logging + logger.setLevel(logging.WARNING) + # Create a test handler to capture log messages import io @@ -3509,12 +3515,13 @@ def test_rownumber_warning_logged(cursor, db_connection): # Verify rownumber functionality still works assert ( - rownumber is None - ), f"Expected rownumber None before fetch, got {rownumber}" + rownumber == -1 + ), f"Expected rownumber -1 before fetch, got {rownumber}" finally: - # Clean up: remove our test handler + # Clean up: remove our test handler and restore level logger.removeHandler(test_handler) + logger.setLevel(original_level) else: # If no logger configured, just test that rownumber works rownumber = cursor.rownumber diff --git a/tests/test_007_logging.py b/tests/test_007_logging.py index 2dabc404..894c7a77 100644 --- a/tests/test_007_logging.py +++ b/tests/test_007_logging.py @@ -2,29 +2,13 @@ import os import pytest import glob -from mssql_python.logging_config import setup_logging, get_logger, LoggingManager +from mssql_python.logging import logger, FINE, FINER, FINEST, setup_logging, get_logger def get_log_file_path(): - # Get the LoggingManager singleton instance - manager = LoggingManager() - # If logging is enabled, return the actual log file path - if manager.enabled and manager.log_file: - return manager.log_file - # For fallback/cleanup, try to find existing log files in the logs directory - repo_root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - log_dir = os.path.join(repo_root_dir, "mssql_python", "logs") - os.makedirs(log_dir, exist_ok=True) - - # Try to find existing log files - log_files = glob.glob(os.path.join(log_dir, "mssql_python_trace_*.log")) - if log_files: - # Return the most recently created log file - return max(log_files, key=os.path.getctime) - - # Fallback to default pattern - pid = os.getpid() - return os.path.join(log_dir, f"mssql_python_trace_{pid}.log") + """Get the current log file path from the logger""" + # The new logger always has a log_file property + return logger.log_file @pytest.fixture @@ -32,25 +16,19 @@ def cleanup_logger(): """Cleanup logger & log files before and after each test""" def cleanup(): - # Get the LoggingManager singleton instance - manager = LoggingManager() - logger = get_logger() - if logger is not None: - logger.handlers.clear() - - # Try to remove the actual log file if it exists + # Disable logging by setting level to CRITICAL + logger.setLevel(logging.CRITICAL) + + # Remove old log file if it exists try: log_file_path = get_log_file_path() if os.path.exists(log_file_path): os.remove(log_file_path) except: pass # Ignore errors during cleanup - - # Reset the LoggingManager instance - manager._enabled = False - manager._initialized = False - manager._logger = None - manager._log_file = None + + # Reset handlers to create a new log file + logger.reset_handlers() # Perform cleanup before the test cleanup() @@ -59,126 +37,183 @@ def cleanup(): cleanup() -def test_no_logging(cleanup_logger): - """Test that logging is off by default""" +def test_logging_disabled_by_default(cleanup_logger): + """Test that logging is disabled by default (level=CRITICAL)""" + try: + # By default, logger should be at CRITICAL level (effectively disabled) + assert logger.getLevel() == logging.CRITICAL + assert not logger.isEnabledFor(FINE) + assert not logger.isEnabledFor(FINER) + assert not logger.isEnabledFor(FINEST) + except Exception as e: + pytest.fail(f"Logging not disabled by default. Error: {e}") + + +def test_enable_logging_fine(cleanup_logger): + """Test enabling logging at FINE level""" try: - # Get the LoggingManager singleton instance - manager = LoggingManager() - logger = get_logger() - assert logger is None - assert manager.enabled == False + logger.setLevel(FINE) + assert logger.getLevel() == FINE + assert logger.isEnabledFor(FINE) + assert not logger.isEnabledFor(FINER) # FINER is more detailed, should be disabled + assert not logger.isEnabledFor(FINEST) # FINEST is most detailed, should be disabled except Exception as e: - pytest.fail(f"Logging not off by default. Error: {e}") + pytest.fail(f"Failed to enable FINE logging: {e}") -def test_setup_logging(cleanup_logger): - """Test if logging is set up correctly""" +def test_enable_logging_finer(cleanup_logger): + """Test enabling logging at FINER level""" try: - setup_logging() # This must enable logging - logger = get_logger() - assert logger is not None - # Fix: Check for the correct logger name - assert logger == logging.getLogger("mssql_python") - assert logger.level == logging.DEBUG # DEBUG level + logger.setLevel(FINER) + assert logger.getLevel() == FINER + assert logger.isEnabledFor(FINE) # FINE is less detailed, should be enabled + assert logger.isEnabledFor(FINER) + assert not logger.isEnabledFor(FINEST) # FINEST is more detailed, should be disabled except Exception as e: - pytest.fail(f"Logging setup failed: {e}") + pytest.fail(f"Failed to enable FINER logging: {e}") -def test_logging_in_file_mode(cleanup_logger): +def test_enable_logging_finest(cleanup_logger): + """Test enabling logging at FINEST level""" + try: + logger.setLevel(FINEST) + assert logger.getLevel() == FINEST + assert logger.isEnabledFor(FINE) + assert logger.isEnabledFor(FINER) + assert logger.isEnabledFor(FINEST) # All levels enabled + except Exception as e: + pytest.fail(f"Failed to enable FINEST logging: {e}") + + +def test_logging_to_file(cleanup_logger): """Test if logging works correctly in file mode""" try: - setup_logging() - logger = get_logger() - assert logger is not None - # Log a test message - test_message = "Testing file logging mode" - logger.info(test_message) - # Check if the log file is created and contains the test message + # Set to FINEST to capture both FINE and INFO messages + logger.setLevel(FINEST) + + # Log test messages at different levels + test_message_fine = "Testing FINE level logging" + test_message_info = "Testing INFO level logging" + + logger.fine(test_message_fine) + logger.info(test_message_info) + + # Check if the log file is created and contains the test messages log_file_path = get_log_file_path() assert os.path.exists(log_file_path), "Log file not created" - # open the log file and check its content + + # Open the log file and check its content with open(log_file_path, "r") as f: log_content = f.read() - assert test_message in log_content, "Log message not found in log file" + + assert test_message_fine in log_content, "FINE message not found in log file" + assert test_message_info in log_content, "INFO message not found in log file" + assert "[Python]" in log_content, "Python prefix not found in log file" except Exception as e: - pytest.fail(f"Logging in file mode failed: {e}") + pytest.fail(f"Logging to file failed: {e}") -def test_logging_in_stdout_mode(cleanup_logger, capsys): - """Test if logging works correctly in stdout mode""" +def test_password_sanitization(cleanup_logger): + """Test that passwords are sanitized in log messages""" try: - setup_logging("stdout") - logger = get_logger() - assert logger is not None - # Log a test message - test_message = "Testing file + stdout logging mode" - logger.info(test_message) - # Check if the log file is created and contains the test message + # Set to FINEST to ensure FINE messages are logged + logger.setLevel(FINEST) + + # Log a message with a password + test_message = "Connection string: Server=localhost;PWD=secret123;Database=test" + logger.fine(test_message) + + # Check if the log file contains the sanitized message log_file_path = get_log_file_path() - assert os.path.exists(log_file_path), "Log file not created in file+stdout mode" with open(log_file_path, "r") as f: log_content = f.read() - assert test_message in log_content, "Log message not found in log file" - # Check if the message is printed to stdout - captured_stdout = capsys.readouterr().out - assert test_message in captured_stdout, "Log message not found in stdout" + + assert "PWD=***" in log_content, "Password not sanitized in log file" + assert "secret123" not in log_content, "Password leaked in log file" except Exception as e: - pytest.fail(f"Logging in stdout mode failed: {e}") + pytest.fail(f"Password sanitization test failed: {e}") -def test_python_layer_prefix(cleanup_logger): - """Test that Python layer logs have the correct prefix""" +def test_trace_id_generation(cleanup_logger): + """Test that trace IDs are generated correctly""" try: - setup_logging() - logger = get_logger() - assert logger is not None + # Generate trace IDs + trace_id1 = logger.generate_trace_id() + trace_id2 = logger.generate_trace_id("Connection") + trace_id3 = logger.generate_trace_id() + + # Check format: PID_ThreadID_Counter + import re + pattern = r'^\d+_\d+_\d+$' + assert re.match(pattern, trace_id1), f"Trace ID format invalid: {trace_id1}" + + # Check format with prefix: Prefix_PID_ThreadID_Counter + pattern_with_prefix = r'^Connection_\d+_\d+_\d+$' + assert re.match(pattern_with_prefix, trace_id2), f"Trace ID with prefix format invalid: {trace_id2}" + + # Check that trace IDs are unique (counter increments) + assert trace_id1 != trace_id3, "Trace IDs should be unique" + except Exception as e: + pytest.fail(f"Trace ID generation test failed: {e}") - # Log a test message - test_message = "This is a Python layer test message" - logger.info(test_message) - # Check if the log file contains the message with [Python Layer log] prefix +def test_log_file_location(cleanup_logger): + """Test that log file is created in current working directory""" + try: + logger.setLevel(FINE) + logger.fine("Test message") + log_file_path = get_log_file_path() - with open(log_file_path, "r") as f: - log_content = f.read() - - # The logged message should have the Python Layer prefix - assert "[Python Layer log]" in log_content, "Python Layer log prefix not found" - assert test_message in log_content, "Test message not found in log file" + + # Log file should be in current working directory, not package directory + cwd = os.getcwd() + assert log_file_path.startswith(cwd), f"Log file not in CWD: {log_file_path}" + + # Check filename format: mssql_python_trace_YYYYMMDD_HHMMSS_PID.log + import re + filename = os.path.basename(log_file_path) + pattern = r'^mssql_python_trace_\d{8}_\d{6}_\d+\.log$' + assert re.match(pattern, filename), f"Log filename format invalid: {filename}" except Exception as e: - pytest.fail(f"Python layer prefix test failed: {e}") + pytest.fail(f"Log file location test failed: {e}") def test_different_log_levels(cleanup_logger): """Test that different log levels work correctly""" try: - setup_logging() - logger = get_logger() - assert logger is not None - + logger.setLevel(FINEST) # Enable all levels + # Log messages at different levels - debug_msg = "This is a DEBUG message" + finest_msg = "This is a FINEST message" + finer_msg = "This is a FINER message" + fine_msg = "This is a FINE message" info_msg = "This is an INFO message" warning_msg = "This is a WARNING message" error_msg = "This is an ERROR message" - - logger.debug(debug_msg) + + logger.finest(finest_msg) + logger.finer(finer_msg) + logger.fine(fine_msg) logger.info(info_msg) logger.warning(warning_msg) logger.error(error_msg) - + # Check if the log file contains all messages log_file_path = get_log_file_path() with open(log_file_path, "r") as f: log_content = f.read() - - assert debug_msg in log_content, "DEBUG message not found in log file" + + assert finest_msg in log_content, "FINEST message not found in log file" + assert finer_msg in log_content, "FINER message not found in log file" + assert fine_msg in log_content, "FINE message not found in log file" assert info_msg in log_content, "INFO message not found in log file" assert warning_msg in log_content, "WARNING message not found in log file" assert error_msg in log_content, "ERROR message not found in log file" - - # Also check for level indicators in the log - assert "DEBUG" in log_content, "DEBUG level not found in log file" + + # Check for level indicators in the log + assert "FINEST" in log_content, "FINEST level not found in log file" + assert "FINER" in log_content, "FINER level not found in log file" + assert "FINE" in log_content, "FINE level not found in log file" assert "INFO" in log_content, "INFO level not found in log file" assert "WARNING" in log_content, "WARNING level not found in log file" assert "ERROR" in log_content, "ERROR level not found in log file" @@ -186,138 +221,186 @@ def test_different_log_levels(cleanup_logger): pytest.fail(f"Log levels test failed: {e}") +def test_backward_compatibility_setup_logging(cleanup_logger): + """Test that deprecated setup_logging() function still works""" + try: + # The old setup_logging() should still work for backward compatibility + setup_logging('file', logging.DEBUG) + + # Logger should be enabled + assert logger.isEnabledFor(FINE) + + # Test logging works + test_message = "Testing backward compatibility" + logger.info(test_message) + + log_file_path = get_log_file_path() + with open(log_file_path, "r") as f: + log_content = f.read() + + assert test_message in log_content + except Exception as e: + pytest.fail(f"Backward compatibility test failed: {e}") + + def test_singleton_behavior(cleanup_logger): - """Test that LoggingManager behaves as a singleton""" + """Test that logger behaves as a module-level singleton""" try: - # Create multiple instances of LoggingManager - manager1 = LoggingManager() - manager2 = LoggingManager() + # Import logger multiple times + from mssql_python.logging import logger as logger1 + from mssql_python.logging import logger as logger2 # They should be the same instance - assert manager1 is manager2, "LoggingManager instances are not the same" + assert logger1 is logger2, "Logger instances are not the same" # Enable logging through one instance - manager1._enabled = True + logger1.setLevel(logging.DEBUG) # The other instance should reflect this change - assert manager2.enabled == True, "Singleton state not shared between instances" + assert logger2.level == logging.DEBUG, "Logger state not shared between instances" # Reset for cleanup - manager1._enabled = False + logger1.setLevel(logging.NOTSET) except Exception as e: pytest.fail(f"Singleton behavior test failed: {e}") def test_timestamp_in_log_filename(cleanup_logger): - """Test that log filenames include timestamps""" + """Test that log filenames include timestamp and PID""" + from mssql_python.logging import logger try: - setup_logging() + # Enable logging + logger.setLevel(logging.DEBUG) + logger.debug("Test message to create log file") # Get the log file path log_file_path = get_log_file_path() + assert log_file_path is not None, "No log file found" + filename = os.path.basename(log_file_path) - # Extract parts of the filename - parts = filename.split("_") - # The filename should follow the pattern: mssql_python_trace_YYYYMMDD_HHMMSS_PID.log - # Fix: Account for the fact that "mssql_python" contains an underscore - assert parts[0] == "mssql", "Incorrect filename prefix part 1" - assert parts[1] == "python", "Incorrect filename prefix part 2" - assert parts[2] == "trace", "Incorrect filename part" - - # Check date format (YYYYMMDD) - date_part = parts[3] - assert ( - len(date_part) == 8 and date_part.isdigit() - ), "Date format incorrect in filename" - - # Check time format (HHMMSS) - time_part = parts[4] - assert ( - len(time_part) == 6 and time_part.isdigit() - ), "Time format incorrect in filename" - - # Process ID should be the last part before .log - pid_part = parts[5].split(".")[0] - assert pid_part.isdigit(), "Process ID not found in filename" - except Exception as e: - pytest.fail(f"Timestamp in filename test failed: {e}") - - -def test_invalid_logging_mode(cleanup_logger): - """Test that invalid logging modes raise ValueError (Lines 130-138).""" - from mssql_python.logging_config import LoggingManager + # Example: mssql_python_trace_20251031_102517_90898.log + assert filename.startswith("mssql_python_trace_"), "Incorrect filename prefix" + assert filename.endswith(".log"), "Incorrect filename suffix" - # Test invalid mode "invalid" - should trigger line 134 - manager = LoggingManager() - with pytest.raises(ValueError, match="Invalid logging mode: invalid"): - manager.setup(mode="invalid") + # Extract the parts between prefix and suffix + middle_part = filename[len("mssql_python_trace_"):-len(".log")] + parts = middle_part.split("_") - # Test another invalid mode "console" - should also trigger line 134 - with pytest.raises(ValueError, match="Invalid logging mode: console"): - manager.setup(mode="console") + # Should have exactly 3 parts: YYYYMMDD, HHMMSS, PID + assert len(parts) == 3, f"Expected 3 parts in filename, got {len(parts)}: {parts}" - # Test invalid mode "both" - should also trigger line 134 - with pytest.raises(ValueError, match="Invalid logging mode: both"): - manager.setup(mode="both") + # Validate parts + date_part, time_part, pid_part = parts + assert len(date_part) == 8 and date_part.isdigit(), f"Date part '{date_part}' is not valid (expected YYYYMMDD)" + assert len(time_part) == 6 and time_part.isdigit(), f"Time part '{time_part}' is not valid (expected HHMMSS)" + assert pid_part.isdigit(), f"PID part '{pid_part}' is not numeric" - # Test empty string mode - should trigger line 134 - with pytest.raises(ValueError, match="Invalid logging mode: "): - manager.setup(mode="") + # PID should match current process ID + assert int(pid_part) == os.getpid(), "PID in filename doesn't match current process" + except Exception as e: + pytest.fail(f"Timestamp in filename test failed: {e}") - # Test None as mode (will become string "None") - should trigger line 134 - with pytest.raises(ValueError, match="Invalid logging mode: None"): - manager.setup(mode=str(None)) +def test_invalid_logging_level(cleanup_logger): + """Test that invalid logging levels are handled correctly.""" + from mssql_python.logging import logger -def test_valid_logging_modes_for_comparison(cleanup_logger): - """Test that valid logging modes work correctly for comparison.""" - from mssql_python.logging_config import LoggingManager + # Test invalid level type - should raise TypeError or ValueError + with pytest.raises((TypeError, ValueError)): + logger.setLevel("invalid_level") - # Test valid mode "file" - should not raise exception - manager = LoggingManager() - try: - logger = manager.setup(mode="file") - assert logger is not None - assert manager.enabled is True - except ValueError: - pytest.fail("Valid mode 'file' should not raise ValueError") - - # Reset manager for next test - manager._enabled = False - manager._initialized = False - manager._logger = None - manager._log_file = None - - # Test valid mode "stdout" - should not raise exception + # Test negative level - Python logging allows this but we can test boundaries try: - logger = manager.setup(mode="stdout") - assert logger is not None - assert manager.enabled is True - except ValueError: - pytest.fail("Valid mode 'stdout' should not raise ValueError") + logger.setLevel(-1) + # If it doesn't raise, verify it's set + assert logger.level == -1 or logger.level >= 0 + except (TypeError, ValueError): + pass # Some implementations may reject negative levels + # Test extremely high level + try: + logger.setLevel(999999) + assert logger.level == 999999 + except (TypeError, ValueError): + pass # Some implementations may have max levels + + +def test_valid_logging_levels_for_comparison(cleanup_logger): + """Test that valid logging levels work correctly.""" + from mssql_python.logging import logger, FINE, FINER, FINEST + + # Test standard Python levels + valid_levels = [ + logging.DEBUG, + logging.INFO, + logging.WARNING, + logging.ERROR, + logging.CRITICAL, + ] + + for level in valid_levels: + try: + logger.setLevel(level) + assert logger.level == level, f"Level {level} not set correctly" + except Exception as e: + pytest.fail(f"Valid level {level} should not raise exception: {e}") + + # Test custom JDBC-style levels + custom_levels = [FINEST, FINER, FINE] + for level in custom_levels: + try: + logger.setLevel(level) + assert logger.level == level, f"Custom level {level} not set correctly" + except Exception as e: + pytest.fail(f"Valid custom level {level} should not raise exception: {e}") -def test_logging_mode_validation_error_message_format(cleanup_logger): - """Test that the error message format for invalid modes is correct.""" - from mssql_python.logging_config import LoggingManager - - manager = LoggingManager() + # Reset + logger.setLevel(logging.NOTSET) - # Test the exact error message format from line 134 - invalid_modes = ["invalid", "debug", "console", "stderr", "syslog"] - for invalid_mode in invalid_modes: - with pytest.raises(ValueError) as exc_info: - manager.setup(mode=invalid_mode) +def test_logging_level_hierarchy(cleanup_logger): + """Test that logging level hierarchy works correctly.""" + from mssql_python.logging import logger, FINE, FINER, FINEST + import io - # Verify the error message format matches line 134 - expected_message = f"Invalid logging mode: {invalid_mode}" - assert str(exc_info.value) == expected_message + # Create a string buffer to capture log output + log_buffer = io.StringIO() + handler = logging.StreamHandler(log_buffer) + handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) + logger.addHandler(handler) - # Reset manager state for next iteration - manager._enabled = False - manager._initialized = False - manager._logger = None - manager._log_file = None + try: + # Set level to INFO - should only show INFO and above + logger.setLevel(logging.INFO) + + logger.debug("Debug message") # Should NOT appear + logger.info("Info message") # Should appear + logger.warning("Warning message") # Should appear + + output = log_buffer.getvalue() + assert "Debug message" not in output, "Debug message should not appear at INFO level" + assert "Info message" in output, "Info message should appear at INFO level" + assert "Warning message" in output, "Warning message should appear at INFO level" + + # Clear buffer + log_buffer.truncate(0) + log_buffer.seek(0) + + # Set to FINEST - should show everything + logger.setLevel(FINEST) + logger.log(FINEST, "Finest message") + logger.log(FINER, "Finer message") + logger.log(FINE, "Fine message") + logger.debug("Debug message") + + output = log_buffer.getvalue() + assert "Finest message" in output, "Finest message should appear at FINEST level" + assert "Finer message" in output, "Finer message should appear at FINEST level" + assert "Fine message" in output, "Fine message should appear at FINEST level" + assert "Debug message" in output, "Debug message should appear at FINEST level" + + finally: + logger.removeHandler(handler) + logger.setLevel(logging.NOTSET)