data-profiler

name	data-profiler
description	Profile construction data to understand characteristics, distributions, quality metrics, and patterns. Essential for data quality assessment and ETL planning.
homepage	https://datadrivenconstruction.io
metadata	{"openclaw":{"emoji":"🏷️","os":["darwin","linux","win32"],"homepage":"https://datadrivenconstruction.io","requires":{"bins":["python3"]}}}

Data Profiler for Construction

Overview

Analyze construction data to understand its characteristics, distributions, quality, and patterns. Essential for data quality assessment, ETL planning, and identifying data issues before they impact projects.

Business Case

Before using any construction data, you need to understand:

• What data types are present
• Distribution of values
• Missing data patterns
• Anomalies and outliers
• Referential integrity issues

This skill profiles data to answer these questions and provides actionable insights.

Technical Implementation

from dataclasses import dataclass, field
from typing import List, Dict, Any, Optional, Tuple
 pandas  pd
 numpy  np
 datetime  datetime
 json


 :
    name: 
    data_type: 
    inferred_type:   
    total_count: 
    null_count: 
    null_percentage: 
    unique_count: 
    uniqueness_ratio: 
    
    min_value: [] = 
    max_value: [] = 
    mean_value: [] = 
    median_value: [] = 
    std_dev: [] = 
    
    min_length: [] = 
    max_length: [] = 
    avg_length: [] = 
    
    top_values: [[, ]] = field(default_factory=)
    
    common_patterns: [] = field(default_factory=)
    
    quality_issues: [] = field(default_factory=)


 :
    source_name: 
    row_count: 
    column_count: 
    columns: [ColumnProfile]
    duplicate_rows: 
    memory_usage: 
    profiled_at: datetime
    quality_score: 
    recommendations: []

 :
    

    
    CONSTRUCTION_PATTERNS = {
        : ,
        : ,
        : ,
        : ,
        : ,
        : ,
        : ,
        : ,
        : ,
        : ,
    }

    
    COLUMN_TYPE_HINTS = {
        : [, , , ],
        : [, , , , , ],
        : [, , , , , ],
        : [, , , ],
        : [, , , ],
        : [, , , , ],
        : [, , , , ],
    }

     ():
        .profiles: [, DataProfile] = {}

     () -> DataProfile:
        
        columns = []

         col  df.columns:
            col_profile = ._profile_column(df[col], col)
            columns.append(col_profile)

        
        duplicate_rows = (df) - (df.drop_duplicates())

        
        memory_bytes = df.memory_usage(deep=).()
         memory_bytes < :
            memory_usage = 
         memory_bytes < **:
            memory_usage = 
        :
            memory_usage = 

        
        quality_score = ._calculate_quality_score(columns)

        
        recommendations = ._generate_recommendations(columns, df)

        profile = DataProfile(
            source_name=source_name,
            row_count=(df),
            column_count=(df.columns),
            columns=columns,
            duplicate_rows=duplicate_rows,
            memory_usage=memory_usage,
            profiled_at=datetime.now(),
            quality_score=quality_score,
            recommendations=recommendations
        )

        .profiles[source_name] = profile
         profile

     () -> ColumnProfile:
        
        total_count = (series)
        null_count = series.isnull().()
        null_percentage = (null_count / total_count * )  total_count >   

        
        non_null = series.dropna()
        unique_count = non_null.nunique()
        uniqueness_ratio = unique_count / (non_null)  (non_null) >   

        profile = ColumnProfile(
            name=name,
            data_type=(series.dtype),
            inferred_type=._infer_construction_type(series, name),
            total_count=total_count,
            null_count=null_count,
            null_percentage=(null_percentage, ),
            unique_count=unique_count,
            uniqueness_ratio=(uniqueness_ratio, )
        )

        
         pd.api.types.is_numeric_dtype(series):
            profile.min_value = (non_null.())  (non_null) >   
            profile.max_value = (non_null.())  (non_null) >   
            profile.mean_value = (non_null.mean())  (non_null) >   
            profile.median_value = (non_null.median())  (non_null) >   
            profile.std_dev = (non_null.std())  (non_null) >   

            
             (non_null) >   profile.std_dev:
                outliers = non_null[(non_null - profile.mean_value) >  * profile.std_dev]
                 (outliers) > :
                    profile.quality_issues.append()

            
             (hint  name.lower()  hint  [, , , ]):
                negatives = (non_null < ).()
                 negatives > :
                    profile.quality_issues.append()

        
         pd.api.types.is_object_dtype(series)  pd.api.types.is_string_dtype(series):
            str_series = non_null.astype()
            lengths = str_series..()
            profile.min_length = (lengths.())  (lengths) >   
            profile.max_length = (lengths.())  (lengths) >   
            profile.avg_length = (lengths.mean())  (lengths) >   

            
            profile.common_patterns = ._detect_patterns(str_series)

        
         (non_null) > :
            value_counts = non_null.value_counts().head()
            profile.top_values = ((value_counts.index.tolist(), value_counts.values.tolist()))

        
         null_percentage > :
            profile.quality_issues.append()
         uniqueness_ratio ==   total_count > :
            profile.quality_issues.append()
         uniqueness_ratio <   unique_count > :
            profile.quality_issues.append()

         profile

     () -> :
        
        name_lower = name.lower()

        
         type_name, hints  .COLUMN_TYPE_HINTS.items():
             (hint  name_lower  hint  hints):
                 type_name

        
        non_null = series.dropna().astype()
         (non_null) == :
             

        sample = non_null.head()

         pattern_name, pattern  .CONSTRUCTION_PATTERNS.items():
            matches = sample..(pattern, na=).()
             matches / (sample) > :
                 pattern_name

        
         pd.api.types.is_numeric_dtype(series):
             
         pd.api.types.is_datetime64_any_dtype(series):
             
        :
             

     () -> []:
        
        patterns_found = []

        sample = str_series.head()

         pattern_name, pattern  .CONSTRUCTION_PATTERNS.items():
            matches = sample..(pattern, na=).()
             matches / (sample) > :
                patterns_found.append()

         patterns_found[:]

     () -> :
        
          columns:
             

        scores = []

         col  columns:
            col_score = 

            
            col_score -= (col.null_percentage, )

            
            col_score -= (col.quality_issues) * 

            scores.append((col_score, ))

         ((scores) / (scores), )

     () -> []:
        
        recommendations = []

        
        high_null = [c  c  columns  c.null_percentage > ]
         high_null:
            recommendations.append(
                
                
            )

        
         col  columns:
               col.name.lower()  col.uniqueness_ratio < :
                recommendations.append(
                    
                )

        
         col  columns:
             col.inferred_type  [, ]  col.data_type == :
                recommendations.append(
                    
                )

        
         col  columns:
             col.inferred_type ==   col.data_type == :
                recommendations.append(
                    
                )

         recommendations

     () -> :
        
         {
            : profile.source_name,
            : profile.row_count,
            : profile.column_count,
            : profile.duplicate_rows,
            : profile.memory_usage,
            : profile.profiled_at.isoformat(),
            : profile.quality_score,
            : profile.recommendations,
            : [
                {
                    : c.name,
                    : c.data_type,
                    : c.inferred_type,
                    : c.null_percentage,
                    : c.unique_count,
                    : c.quality_issues,
                    : c.top_values[:]
                }
                 c  profile.columns
            ]
        }

     () -> :
        
        report = [, ]
        report.append()
        report.append()
        report.append()

        
        report.append()
        report.append()
        report.append()
        report.append()
        report.append()
        report.append()

        
         profile.recommendations:
            report.append()
             rec  profile.recommendations:
                report.append()
            report.append()

        
        report.append()
        report.append()
        report.append()
        report.append()

         col  profile.columns:
            issues = (col.quality_issues)
            report.append(
                
                
            )

        
        report.append()
        report.append()

         col  profile.columns:
            report.append()
            report.append()
            report.append()
            report.append()

             col.min_value   :
                report.append()
                report.append()

             col.min_length   :
                report.append()

             col.top_values:
                report.append()

             col.common_patterns:
                report.append()

             col.quality_issues:
                report.append()

         .join(report)

     () -> :
        
        comparison = {
            : [profile1.source_name, profile2.source_name],
            : profile2.row_count - profile1.row_count,
            : profile2.quality_score - profile1.quality_score,
            : [],
            : [],
            : [],
            : []
        }

        cols1 = {c.name: c  c  profile1.columns}
        cols2 = {c.name: c  c  profile2.columns}

        
        comparison[] = [n  n  cols2  n   cols1]
        comparison[] = [n  n  cols1  n   cols2]

        
         name  cols1:
             name  cols2:
                c1, c2 = cols1[name], cols2[name]

                 c1.data_type != c2.data_type:
                    comparison[].append({
                        : name,
                        : c1.data_type,
                        : c2.data_type
                    })

                null_change = c2.null_percentage - c1.null_percentage
                 (null_change) > :
                    comparison[].append({
                        : name,
                        : null_change
                    })

         comparison