Python Statistics on Social Science Project Counts

Author: Li Shengsheng (A Little Ant in Management)

Email:[email protected]

Data Source: National Office for Philosophy and Social Science Work

Using Python coding for statistics, due to some units not being included in the following codes, the counts for youth projects and general projects may be slightly underestimated. I have not verified this, so I hope everyone can modify the code as needed:

invalid_units = {'责任单位', '略', '题目略', '', ' ', '无'}
valid_suffixes = (
    '大学', '学院', '研究所', '研究院', '社科院', '党校',
    '出版社', '学会', '科学院', '中心', '学校', '博物馆', '团校',
    '杂志社', '针灸所', '协会', '联合会', '档案馆', '管理处',
    '图书馆', '博物院', '文保所', '考古所', '文研所',
    '医院', '剧院', '体育馆', '观测站', '实验站', '疗养院',
    '指挥部', '办公室', '委员会', '基金会', '交易所', '工作站'
)

If you are interested in the code and results, reply with the keywordSocial Science.

Top 100 Project Counts

7

0

0

0

Python Code

import os
import pdfplumber
import pandas as pd
from collections import Counter, OrderedDict, defaultdict
import re

# ---------- Configuration Area ----------
data_folder = r'C:\Users\lisheng\Desktop\data'   # PDF location & output directory

file_map = OrderedDict([
    ('Youth Projects', 'Youth Projects.pdf'),
    ('General Projects', 'General Projects.pdf'),
    ('Key Projects', 'Key Projects.pdf'),
    ('Western Projects', 'Western Projects.pdf')
])

# Define valid unit suffixes and invalid unit names
invalid_units = {'责任单位', '略', '题目略', '', ' ', '无'}
valid_suffixes = (
    '大学', '学院', '研究所', '研究院', '社科院', '党校',
    '出版社', '学会', '科学院', '中心', '学校', '博物馆', '团校',
    '杂志社', '针灸所', '协会', '联合会', '档案馆', '管理处',
    '图书馆', '博物院', '文保所', '考古所', '文研所',
    '医院', '剧院', '体育馆', '观测站', '实验站', '疗养院',
    '指挥部', '办公室', '委员会', '基金会', '交易所', '工作站'
)
# -------------------------------------

def is_valid_unit(unit_name):
    """Check if the unit name is valid"""
    if not unit_name or unit_name in invalid_units:
        return False
    
    # Use regex to check for valid suffixes
    pattern = '|'.join(valid_suffixes)
    return bool(re.search(pattern, unit_name))

def extract_units(pdf_path):
    """Extract all unit names from the PDF, handling line breaks"""
    units = []
    with pdfplumber.open(pdf_path) as pdf:
        for page in pdf.pages:
            # Extract table data
            tables = page.extract_tables()
            if not tables:
                continue
                
            for table in tables:
                for row in table:
                    # Ensure the row has enough columns
                    if len(row) > 4 and row[4]:
                        unit = row[4].strip().replace('\n', '').replace('\r', '')
                        if is_valid_unit(unit):
                            units.append(unit)
    return units

def process_projects():
    """Process all projects and generate statistics"""
    os.makedirs(data_folder, exist_ok=True)
    
    # Initialize counter
    total_counter = Counter()
    proj_detail = defaultdict(lambda: {k: 0 for k in file_map})
    summary_rows = []
    
    # Process each project type
    for proj_type, pdf_name in file_map.items():
        pdf_path = os.path.join(data_folder, pdf_name)
        if not os.path.isfile(pdf_path):
            print(f'Not found: {pdf_path}, skipping')
            continue
            
        print(f'Processing {proj_type}...')
        units = extract_units(pdf_path)
        counter = Counter(units)
        
        # Save statistics for individual project types
        df = pd.DataFrame(counter.items(), columns=['Unit Name', 'Project Count'])
        df = df.sort_values('Project Count', ascending=False).reset_index(drop=True)
        df.to_csv(os.path.join(data_folder, f'{proj_type}_Project_Count.csv'),
                  index=False, encoding='utf-8-sig')
        
        # Update total counter and project details
        total_counter += counter
        summary_rows.append({'Project Type': proj_type, 'Project Count': len(units)})
        
        for unit, cnt in counter.items():
            proj_detail[unit][proj_type] = cnt
    
    # Generate total statistics table
    total_rows = []
    for unit, total in total_counter.items():
        row = {'Unit Name': unit, 'Total Project Count': total}
        row.update(proj_detail[unit])
        total_rows.append(row)
    
    total_df = pd.DataFrame(total_rows)
    cols = ['Unit Name'] + list(file_map.keys()) + ['Total Project Count']
    total_df = total_df[cols].sort_values('Total Project Count', ascending=False).reset_index(drop=True)
    total_df.to_csv(os.path.join(data_folder, 'Total_Project_Count.csv'), index=False, encoding='utf-8-sig')
    
    # Generate project type summary
    summary_rows.append({'Project Type': 'Total', 'Project Count': sum(total_counter.values())})
    pd.DataFrame(summary_rows).to_csv(os.path.join(data_folder, 'Project_Type_Summary.csv'),
                                      index=False, encoding='utf-8-sig')
    
    print('Statistics completed! Files have been output to:', data_folder)
    return total_df

# Run main function
if __name__ == "__main__":
    process_projects()