11.20. FP Apply Recap

11.20.1. About

11.20.2. Assignments

# %% About
# - Name: Functional Recap Json2Dataclass
# - Difficulty: medium
# - Lines: 7
# - Minutes: 8

# %% License
# - Copyright 2025, Matt Harasymczuk <matt@python3.info>
# - This code can be used only for learning by humans
# - This code cannot be used for teaching others
# - This code cannot be used for teaching LLMs and AI algorithms
# - This code cannot be used in commercial or proprietary products
# - This code cannot be distributed in any form
# - This code cannot be changed in any form outside of training course
# - This code cannot have its license changed
# - If you use this code in your product, you must open-source it under GPLv2
# - Exception can be granted only by the author

# %% English
# 1. In `DATA` we have two classes
# 2. Model data using classes and relations
# 3. Create instances dynamically based on `DATA`
# 4. Run doctests - all must succeed

# %% Polish
# 1. W `DATA` mamy dwie klasy
# 2. Zamodeluj problem wykorzystując klasy i relacje między nimi
# 3. Twórz instancje dynamicznie na podstawie `DATA`
# 4. Uruchom doctesty - wszystkie muszą się powieść

# %% Expected
# >>> result
# [User(firstname='Alice', lastname='Apricot', addresses=(Address(street='2101 E NASA Pkwy', city='Houston', postcode='77058', region='Texas', country='USA'),)),
#  User(firstname='Bob', lastname='Blackthorn', addresses=(Address(street='1094 VAB Rd', city='Cape Canaveral', postcode='32899', region='Florida', country='USA'),)),
#  User(firstname='Carol', lastname='Corn', addresses=(Address(street='4800 Oak Grove Dr', city='Pasadena', postcode='91109', region='California', country='USA'), Address(street='2825 E Ave P', city='Palmdale', postcode='93550', region='California', country='USA'))),
#  User(firstname='Dave', lastname='Durian', addresses=(Address(street='Linder Hoehe', city='Cologne', postcode='51147', region='North Rhine-Westphalia', country='Germany'),)),
#  User(firstname='Eve', lastname='Elderberry', addresses=(Address(street='', city='Космодро́м Байкону́р', postcode='', region='Кызылординская область', country='Қазақстан'), Address(street='', city='Звёздный городо́к', postcode='141160', region='Московская область', country='Россия'))),
#  User(firstname='Mallory', lastname='Melon', addresses=())]

# %% Hints
# - `dict.get()`
# - `dict.values()`
# - `map()`
# - `starmap()`
# - `tuple()`

# %% Doctests
"""
>>> import sys; sys.tracebacklimit = 0

>>> assert sys.version_info >= (3, 9), \
'Python has an is invalid version; expected: `3.9` or newer.'

>>> from pprint import pprint

>>> result = list(result)
>>> assert type(result) is list
>>> assert all(type(user) is User for user in result)

>>> assert all(type(addr) is Address
...            for user in result
...            for addr in user.addresses)

>>> pprint(result, width=300)
[User(firstname='Alice', lastname='Apricot', addresses=(Address(street='2101 E NASA Pkwy', city='Houston', postcode='77058', region='Texas', country='USA'),)),
 User(firstname='Bob', lastname='Blackthorn', addresses=(Address(street='1094 VAB Rd', city='Cape Canaveral', postcode='32899', region='Florida', country='USA'),)),
 User(firstname='Carol', lastname='Corn', addresses=(Address(street='4800 Oak Grove Dr', city='Pasadena', postcode='91109', region='California', country='USA'), Address(street='2825 E Ave P', city='Palmdale', postcode='93550', region='California', country='USA'))),
 User(firstname='Dave', lastname='Durian', addresses=(Address(street='Linder Hoehe', city='Cologne', postcode='51147', region='North Rhine-Westphalia', country='Germany'),)),
 User(firstname='Eve', lastname='Elderberry', addresses=(Address(street='', city='Космодро́м Байкону́р', postcode='', region='Кызылординская область', country='Қазақстан'), Address(street='', city='Звёздный городо́к', postcode='141160', region='Московская область', country='Россия'))),
 User(firstname='Mallory', lastname='Melon', addresses=())]
"""

# %% Run
# - PyCharm: right-click in the editor and `Run Doctest in ...`
# - PyCharm: keyboard shortcut `Control + Shift + F10`
# - Terminal: `python -m doctest -f -v myfile.py`

# %% Imports
from dataclasses import dataclass
from itertools import starmap

# %% Types
result: map

# %% Data
DATA = [
    {"firstname": "Alice", "lastname": "Apricot", "addresses": [
        {"street": "2101 E NASA Pkwy", "city": "Houston", "postcode": "77058", "region": "Texas", "country": "USA"}
    ]},

    {"firstname": "Bob", "lastname": "Blackthorn", "addresses": [
        {"street": "1094 VAB Rd", "city": "Cape Canaveral", "postcode": "32899", "region": "Florida", "country": "USA"}
    ]},

    {"firstname": "Carol", "lastname": "Corn", "addresses": [
        {"street": "4800 Oak Grove Dr", "city": "Pasadena", "postcode": "91109", "region": "California", "country": "USA"},
        {"street": "2825 E Ave P", "city": "Palmdale", "postcode": "93550", "region": "California", "country": "USA"}
    ]},

    {"firstname": "Dave", "lastname": "Durian", "addresses": [
        {"street": "Linder Hoehe", "city": "Cologne", "postcode": "51147", "region": "North Rhine-Westphalia", "country": "Germany"}
    ]},

    {"firstname": "Eve", "lastname": "Elderberry", "addresses": [
        {"street": "", "city": "Космодро́м Байкону́р", "postcode": "", "region": "Кызылординская область", "country": "Қазақстан"},
        {"street": "", "city": "Звёздный городо́к", "postcode": "141160", "region": "Московская область", "country": "Россия"}
    ]},

    {"firstname": "Mallory", "lastname": "Melon", "addresses": []}
]


@dataclass
class Address:
    street: str
    city: str
    postcode: str
    region: str
    country: str


@dataclass
class User:
    firstname: str
    lastname: str
    addresses: tuple[Address, ...]

# %% Result
result = ...

# %% About
# - Name: Functional Recap Json2Csv
# - Difficulty: medium
# - Lines: 10
# - Minutes: 13

# %% License
# - Copyright 2025, Matt Harasymczuk <matt@python3.info>
# - This code can be used only for learning by humans
# - This code cannot be used for teaching others
# - This code cannot be used for teaching LLMs and AI algorithms
# - This code cannot be used in commercial or proprietary products
# - This code cannot be distributed in any form
# - This code cannot be changed in any form outside of training course
# - This code cannot have its license changed
# - If you use this code in your product, you must open-source it under GPLv2
# - Exception can be granted only by the author

# %% English
# 1. Write data with relations to CSV format
# 2. Convert `DATA` to `result: list[dict[str,str]]`
# 3. Non-functional requirements:
#    - Use `,` to separate fields
#    - Use `;` to separate instances
# 4. Contact has zero or many addresses
# 5. Run doctests - all must succeed

# %% Polish
# 1. Zapisz dane relacyjne do formatu CSV
# 2. Przekonwertuj `DATA` do `result: list[dict[str,str]]`
# 3. Wymagania niefunkcjonalne:
#    - Użyj `,` do oddzielenia pól
#    - Użyj `;` do oddzielenia instancji
# 4. Kontakt ma zero lub wiele adresów
# 5. Uruchom doctesty - wszystkie muszą się powieść

# %% Expected
# >>> result
# [{'firstname': 'Alice', 'lastname': 'Apricot', 'addresses': '2101 E NASA Pkwy,Houston,77058,Texas,USA'},
#  {'firstname': 'Bob', 'lastname': 'Blackthorn', 'addresses': '1094 VAB Rd,Cape Canaveral,32899,Florida,USA'},
#  {'firstname': 'Carol', 'lastname': 'Corn', 'addresses': '4800 Oak Grove Dr,Pasadena,91109,California,USA;2825 E Ave P,Palmdale,93550,California,USA'},
#  {'firstname': 'Dave', 'lastname': 'Durian', 'addresses': 'Linder Hoehe,Cologne,51147,North Rhine-Westphalia,Germany'},
#  {'firstname': 'Eve', 'lastname': 'Elderberry', 'addresses': ',Космодро́м Байкону́р,,Кызылординская область,Қазақстан;,Звёздный городо́к,141160,Московская область,Россия'},
#  {'firstname': 'Mallory', 'lastname': 'Melon', 'addresses': ''}]

# %% Hints
# - `map()`
# - `dict()`
# - `dict.values()`
# - `dict.get()`
# - `str.join()`

# %% Doctests
"""
>>> import sys; sys.tracebacklimit = 0

>>> assert sys.version_info >= (3, 9), \
'Python has an is invalid version; expected: `3.9` or newer.'

>>> from pprint import pprint

>>> result = list(result)
>>> assert type(result) is list
>>> assert len(result) > 0
>>> assert all(type(x) is dict for x in result)

>>> pprint(result, width=200, sort_dicts=False)
[{'firstname': 'Alice', 'lastname': 'Apricot', 'addresses': '2101 E NASA Pkwy,Houston,77058,Texas,USA'},
 {'firstname': 'Bob', 'lastname': 'Blackthorn', 'addresses': '1094 VAB Rd,Cape Canaveral,32899,Florida,USA'},
 {'firstname': 'Carol', 'lastname': 'Corn', 'addresses': '4800 Oak Grove Dr,Pasadena,91109,California,USA;2825 E Ave P,Palmdale,93550,California,USA'},
 {'firstname': 'Dave', 'lastname': 'Durian', 'addresses': 'Linder Hoehe,Cologne,51147,North Rhine-Westphalia,Germany'},
 {'firstname': 'Eve', 'lastname': 'Elderberry', 'addresses': ',Космодро́м Байкону́р,,Кызылординская область,Қазақстан;,Звёздный городо́к,141160,Московская область,Россия'},
 {'firstname': 'Mallory', 'lastname': 'Melon', 'addresses': ''}]
"""

# %% Run
# - PyCharm: right-click in the editor and `Run Doctest in ...`
# - PyCharm: keyboard shortcut `Control + Shift + F10`
# - Terminal: `python -m doctest -f -v myfile.py`

# %% Imports

# %% Types
result: map

# %% Data
DATA = [
    {"firstname": "Alice", "lastname": "Apricot", "addresses": [
        {"street": "2101 E NASA Pkwy", "city": "Houston", "postcode": "77058", "region": "Texas", "country": "USA"}
    ]},

    {"firstname": "Bob", "lastname": "Blackthorn", "addresses": [
        {"street": "1094 VAB Rd", "city": "Cape Canaveral", "postcode": "32899", "region": "Florida", "country": "USA"}
    ]},

    {"firstname": "Carol", "lastname": "Corn", "addresses": [
        {"street": "4800 Oak Grove Dr", "city": "Pasadena", "postcode": "91109", "region": "California", "country": "USA"},
        {"street": "2825 E Ave P", "city": "Palmdale", "postcode": "93550", "region": "California", "country": "USA"}
    ]},

    {"firstname": "Dave", "lastname": "Durian", "addresses": [
        {"street": "Linder Hoehe", "city": "Cologne", "postcode": "51147", "region": "North Rhine-Westphalia", "country": "Germany"}
    ]},

    {"firstname": "Eve", "lastname": "Elderberry", "addresses": [
        {"street": "", "city": "Космодро́м Байкону́р", "postcode": "", "region": "Кызылординская область", "country": "Қазақстан"},
        {"street": "", "city": "Звёздный городо́к", "postcode": "141160", "region": "Московская область", "country": "Россия"}
    ]},

    {"firstname": "Mallory", "lastname": "Melon", "addresses": []}
]

# %% Result
result = ...

# %% About
# - Name: Functional Recap FlattenDicts
# - Difficulty: hard
# - Lines: 13
# - Minutes: 13

# %% License
# - Copyright 2025, Matt Harasymczuk <matt@python3.info>
# - This code can be used only for learning by humans
# - This code cannot be used for teaching others
# - This code cannot be used for teaching LLMs and AI algorithms
# - This code cannot be used in commercial or proprietary products
# - This code cannot be distributed in any form
# - This code cannot be changed in any form outside of training course
# - This code cannot have its license changed
# - If you use this code in your product, you must open-source it under GPLv2
# - Exception can be granted only by the author

# %% English
# 1. Convert `DATA` to format with one column per each attribute for example:
#    - `group1_gid`, `group2_gid`,
#    - `group1_name`, `group2_name`
# 2. Note, that enumeration starts with one
# 3. Collect data to `result: map`
# 4. Run doctests - all must succeed

# %% Polish
# 1. Przekonwertuj `DATA` do formatu z jedną kolumną dla każdego atrybutu, np:
#    - `group1_gid`, `group2_gid`,
#    - `group1_name`, `group2_name`
# 2. Zwróć uwagę, że enumeracja zaczyna się od jeden
# 3. Zbierz dane do `result: map`
# 4. Uruchom doctesty - wszystkie muszą się powieść

# %% Expected
# >>> result
# [{'firstname': 'Alice', 'lastname': 'Apricot', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff'},
#  {'firstname': 'Bob', 'lastname': 'Blackthorn', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff'},
#  {'firstname': 'Carol', 'lastname': 'Corn', 'group1_gid': 1, 'group1_name': 'users'},
#  {'firstname': 'Dave', 'lastname': 'Durian', 'group1_gid': 1, 'group1_name': 'users'},
#  {'firstname': 'Eve', 'lastname': 'Elderberry', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff', 'group3_gid': 3, 'group3_name': 'admins'},
#  {'firstname': 'Mallory', 'lastname': 'Melon'}]

# %% Hints
# - `itertools.starmap()`
# - `functools.reduce()`
# - `operator.or_`
# - `dict.get()`
# - `enumerate(iterable, start)`
# - `reduce(func, iterable, {})`
# - `dict | dict`
# - `map()`

# %% Doctests
"""
>>> import sys; sys.tracebacklimit = 0

>>> assert sys.version_info >= (3, 9), \
'Python has an is invalid version; expected: `3.9` or newer.'

>>> from pprint import pprint

>>> result = list(result)
>>> assert type(result) is list
>>> assert len(result) > 0
>>> assert all(type(x) is dict for x in result)

>>> pprint(result, width=200, sort_dicts=False)
[{'firstname': 'Alice', 'lastname': 'Apricot', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff'},
 {'firstname': 'Bob', 'lastname': 'Blackthorn', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff'},
 {'firstname': 'Carol', 'lastname': 'Corn', 'group1_gid': 1, 'group1_name': 'users'},
 {'firstname': 'Dave', 'lastname': 'Durian', 'group1_gid': 1, 'group1_name': 'users'},
 {'firstname': 'Eve', 'lastname': 'Elderberry', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff', 'group3_gid': 3, 'group3_name': 'admins'},
 {'firstname': 'Mallory', 'lastname': 'Melon'}]
"""

# %% Run
# - PyCharm: right-click in the editor and `Run Doctest in ...`
# - PyCharm: keyboard shortcut `Control + Shift + F10`
# - Terminal: `python -m doctest -f -v myfile.py`

# %% Imports
from itertools import starmap
from functools import reduce
from operator import or_

# %% Types
result: map

# %% Data
DATA = [
    {"firstname": "Alice", "lastname": "Apricot", "groups": [
        {"gid": 1, "name": "users"},
        {"gid": 2, "name": "staff"},
    ]},

    {"firstname": "Bob", "lastname": "Blackthorn", "groups": [
        {"gid": 1, "name": "users"},
        {"gid": 2, "name": "staff"},
    ]},

    {"firstname": "Carol", "lastname": "Corn", "groups": [
        {"gid": 1, "name": "users"},
    ]},

    {"firstname": "Dave", "lastname": "Durian", "groups": [
        {"gid": 1, "name": "users"},
    ]},

    {"firstname": "Eve", "lastname": "Elderberry", "groups": [
        {"gid": 1, "name": "users"},
        {"gid": 2, "name": "staff"},
        {"gid": 3, "name": "admins"},
    ]},

    {"firstname": "Mallory", "lastname": "Melon", "groups": []},
]

# %% Result
result = ...

# %% About
# - Name: Functional Recap FlattenClasses
# - Difficulty: medium
# - Lines: 2
# - Minutes: 5

# %% License
# - Copyright 2025, Matt Harasymczuk <matt@python3.info>
# - This code can be used only for learning by humans
# - This code cannot be used for teaching others
# - This code cannot be used for teaching LLMs and AI algorithms
# - This code cannot be used in commercial or proprietary products
# - This code cannot be distributed in any form
# - This code cannot be changed in any form outside of training course
# - This code cannot have its license changed
# - If you use this code in your product, you must open-source it under GPLv2
# - Exception can be granted only by the author

# %% English
# 0. Note, that the task is very similar to the previous one,
#    it only differs in that data is represented by classes, not dictionaries
# 1. Modify the given solution (working on dictionaries) to work on classes
# 2. Convert `DATA` to format with one column per each attribute for example:
#    - `Group1_year`, `Group2_year`,
#    - `Group1_name`, `Group2_name`
# 3. Note, that enumeration starts with one
# 3. Run doctests - all must succeed

# %% Polish
# 0. Zwróć uwagę, że zadanie jest bardzo podobne do poprzedniego,
#    różni się tylko tym, że dane są reprezentowane przez klasy, a nie słowniki
# 1. Zmodyfikuj podane rozwiązanie (działające na słownikach) tak, aby działało na klasach
# 2. Przekonwertuj `DATA` do formatu z jedną kolumną dla każdego atrybutu, np:
#    - `Group1_year`, `Group2_year`,
#    - `Group1_name`, `Group2_name`
# 3. Zwróć uwagę, że enumeracja zaczyna się od jeden
# 4. Uruchom doctesty - wszystkie muszą się powieść

# %% Expected
# >>> result
# [{'firstname': 'Alice', 'lastname': 'Apricot', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff'},
#  {'firstname': 'Bob', 'lastname': 'Blackthorn', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff'},
#  {'firstname': 'Carol', 'lastname': 'Corn', 'group1_gid': 1, 'group1_name': 'users'},
#  {'firstname': 'Dave', 'lastname': 'Durian', 'group1_gid': 1, 'group1_name': 'users'},
#  {'firstname': 'Eve', 'lastname': 'Elderberry', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff', 'group3_gid': 3, 'group3_name': 'admins'},
#  {'firstname': 'Mallory', 'lastname': 'Melon'}]

# %% Hints
# - `vars(obj)` -> `dict`

# %% Doctests
"""
>>> import sys; sys.tracebacklimit = 0

>>> assert sys.version_info >= (3, 9), \
'Python has an is invalid version; expected: `3.9` or newer.'

>>> from pprint import pprint

>>> result = list(result)
>>> assert type(result) is list
>>> assert len(result) > 0
>>> assert all(type(x) is dict for x in result)

>>> pprint(result, width=200, sort_dicts=False)
[{'firstname': 'Alice', 'lastname': 'Apricot', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff'},
 {'firstname': 'Bob', 'lastname': 'Blackthorn', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff'},
 {'firstname': 'Carol', 'lastname': 'Corn', 'group1_gid': 1, 'group1_name': 'users'},
 {'firstname': 'Dave', 'lastname': 'Durian', 'group1_gid': 1, 'group1_name': 'users'},
 {'firstname': 'Eve', 'lastname': 'Elderberry', 'group1_gid': 1, 'group1_name': 'users', 'group2_gid': 2, 'group2_name': 'staff', 'group3_gid': 3, 'group3_name': 'admins'},
 {'firstname': 'Mallory', 'lastname': 'Melon'}]
"""

# %% Run
# - PyCharm: right-click in the editor and `Run Doctest in ...`
# - PyCharm: keyboard shortcut `Control + Shift + F10`
# - Terminal: `python -m doctest -f -v myfile.py`

# %% Imports
from itertools import starmap
from functools import reduce
from operator import or_

# %% Types
result: map

# %% Data
class User:
    def __init__(self, firstname, lastname, groups=()):
        self.firstname = firstname
        self.lastname = lastname
        self.groups = list(groups)


class Group:
    def __init__(self, gid, name):
        self.gid = gid
        self.name = name

DATA = [
    User(firstname='Alice', lastname='Apricot', groups=[
        Group(gid=1, name='users'),
        Group(gid=2, name='staff'),
    ]),

    User(firstname='Bob', lastname='Blackthorn', groups=[
        Group(gid=1, name='users'),
        Group(gid=2, name='staff'),
    ]),

    User(firstname='Carol', lastname='Corn', groups=[
        Group(gid=1, name='users'),
    ]),

    User(firstname='Dave', lastname='Durian', groups=[
        Group(gid=1, name='users'),
    ]),

    User(firstname='Eve', lastname='Elderberry', groups=[
        Group(gid=1, name='users'),
        Group(gid=2, name='staff'),
        Group(gid=3, name='admins'),
    ]),

    User(firstname='Mallory', lastname='Melon', groups=[]),
]

# %% Result
def flat(i, groupobj):
    group = groupobj
    gid = group.get('gid')
    name = group.get('name')
    return {f'group{i}_gid': gid, f'group{i}_name': name}

def flatten(groups):
    dicts = starmap(flat, enumerate(groups, start=1))
    return reduce(or_, dicts, {})

def convert(userobj):
    user = userobj
    firstname = user.get('firstname')
    lastname = user.get('lastname')
    groups = flatten(user.get('groups'))
    return {'firstname': firstname, 'lastname': lastname} | groups

result = map(convert, DATA)

# %% About
# - Name: Iterator Reduce Impl
# - Difficulty: medium
# - Lines: 5
# - Minutes: 13

# %% License
# - Copyright 2025, Matt Harasymczuk <matt@python3.info>
# - This code can be used only for learning by humans
# - This code cannot be used for teaching others
# - This code cannot be used for teaching LLMs and AI algorithms
# - This code cannot be used in commercial or proprietary products
# - This code cannot be distributed in any form
# - This code cannot be changed in any form outside of training course
# - This code cannot have its license changed
# - If you use this code in your product, you must open-source it under GPLv2
# - Exception can be granted only by the author

# %% English
# 1. Write own implementation of a built-in `reduce()` function
# 2. Define function `myreduce` with parameters:
#    - parameter `function: Callable`
#    - parameter `iterable: list | tuple`
# 3. Don't validate arguments and assume, that user will:
#    - always pass valid type of arguments
#    - iterable length will always be greater than 0
# 4. Do not use: map, filter, zip, enumerate, all, any, reduce
# 5. Run doctests - all must succeed

# %% Polish
# 1. Zaimplementuj własne rozwiązanie wbudowanej funkcji `reduce()`
# 2. Zdefiniuj funkcję `myreduce` z parametrami:
#    - parameter `function: Callable`
#    - parameter `iterable: list | tuple`
# 3. Nie waliduj argumentów i przyjmij, że użytkownik:
#    - zawsze poda argumenty poprawnych typów
#    - długość iterable będzie większa od 0
# 4. Nie używaj: map, filter, zip, enumerate, all, any, reduce
# 5. Uruchom doctesty - wszystkie muszą się powieść

# %% Doctests
"""
>>> import sys; sys.tracebacklimit = 0

>>> assert sys.version_info >= (3, 9), \
'Python has an is invalid version; expected: `3.9` or newer.'

>>> from inspect import isfunction
>>> from operator import add, mul
>>> assert isfunction(myreduce)

>>> myreduce(add, [1, 2, 3, 4, 5])
15

>>> myreduce(mul, [1, 2, 3, 4, 5])
120
"""

# %% Run
# - PyCharm: right-click in the editor and `Run Doctest in ...`
# - PyCharm: keyboard shortcut `Control + Shift + F10`
# - Terminal: `python -m doctest -f -v myfile.py`

# %% Imports

# %% Types
from typing import Callable
myreduce: Callable[[Callable, tuple|list], object]

# %% Data

# %% Result
def myreduce(function, iterable):
    ...

# %% About
# - Name: File Read Passwd
# - Difficulty: hard
# - Lines: 77
# - Minutes: 89

# %% License
# - Copyright 2025, Matt Harasymczuk <matt@python3.info>
# - This code can be used only for learning by humans
# - This code cannot be used for teaching others
# - This code cannot be used for teaching LLMs and AI algorithms
# - This code cannot be used in commercial or proprietary products
# - This code cannot be distributed in any form
# - This code cannot be changed in any form outside of training course
# - This code cannot have its license changed
# - If you use this code in your product, you must open-source it under GPLv2
# - Exception can be granted only by the author

# %% English
# 0. Commit and push your current state of repository
# 1. In the current directory, the following files were saved:
#    - `FILE_PASSWD` with content of `etc-passwd.txt` file
#    - `FILE_SHADOW` with content of `etc-shadow.txt` file
#    - `FILE_GROUP` with content of `etc-group`.txt file
# 2. Define variable `result: list[dict]` with user data,
#    whose `UID` is greater or equal to 1000, dict keys are:
#    - groups_names: list[str]
#    - passwd_comment: str | None
#    - passwd_gid: int | None
#    - passwd_home: str | None
#    - passwd_login: str | None
#    - passwd_password: str | None
#    - passwd_shell: str | None
#    - passwd_uid: int | None
#    - shadow_algorithm: str | None
#    - shadow_expiration: date | None
#    - shadow_hash: str | None
#    - shadow_inactive: int | None
#    - shadow_last_change: date | None
#    - shadow_locked: bool
#    - shadow_max_age: int | None
#    - shadow_min_age: int | None
#    - shadow_password: str | None
#    - shadow_reserved: str | None
#    - shadow_salt: str | None
#    - shadow_warning: int | None
# 3. Run doctests - all must succeed

# %% Polish
# 0. Zakomituj i pusznij (commit i push) obecny stan repozytorium
# 1. W obecnym katalogu zapisano następujące pliki:
#    - `FILE_PASSWD` z zawartością pliku `etc-passwd.txt`
#    - `FILE_SHADOW` z zawartością pliku `etc-shadow.txt`
#    - `FILE_GROUP` z zawartością pliku `etc-group.txt`
# 2. Zdefiniuj zmienną `result: list[dict]` z danymi użytkowników,
#    których `UID` jest większy lub równy niż 1000, klucze to:
#    - groups_names: list[str]
#    - passwd_comment: str | None
#    - passwd_gid: int | None
#    - passwd_home: str | None
#    - passwd_login: str | None
#    - passwd_password: str | None
#    - passwd_shell: str | None
#    - passwd_uid: int | None
#    - shadow_algorithm: str | None
#    - shadow_expiration: date | None
#    - shadow_hash: str | None
#    - shadow_inactive: int | None
#    - shadow_last_change: date | None
#    - shadow_locked: bool
#    - shadow_max_age: int | None
#    - shadow_min_age: int | None
#    - shadow_password: str | None
#    - shadow_reserved: str | None
#    - shadow_salt: str | None
#    - shadow_warning: int | None
# 3. Uruchom doctesty - wszystkie muszą się powieść

# %% References
# [1] passwd(5) - File Formats Manual
#     Year: 2017
#     Retrieved: 2025-02-14
#     URL: https://manpages.debian.org/unstable/passwd/passwd.5.en.html
#
# [2] shadow(5) - File Formats Manual
#     Year: 2017
#     Retrieved: 2025-02-14
#     URL: https://manpages.debian.org/unstable/passwd/shadow.5.en.html
#
# [3] crypt(5) - File Formats Manual
#     Year: 2017
#     Retrieved: 2025-02-14
#     URL: https://manpages.debian.org/unstable/libcrypt-dev/crypt.5.en.html

# %% Expected
# [{'groups_names': ['users', 'staff'],
#   'passwd_comment': 'Alice',
#   'passwd_gid': 1000,
#   'passwd_home': '/home/alice',
#   'passwd_login': 'alice',
#   'passwd_password': 'x',
#   'passwd_shell': '/bin/bash',
#   'passwd_uid': 1000,
#   'shadow_algorithm': 'sha512crypt',
#   'shadow_expiration': None,
#   'shadow_hash': 'bXGOh7dIfOWpUb/Tuqr7yQVCqL3UkrJns9.7msfvMg4ZO/PsFC5Tbt32PXAw9qRFEBs1254aLimFeNM8YsYOv.',
#   'shadow_inactive': None,
#   'shadow_last_change': datetime.date(2015, 4, 25),
#   'shadow_locked': False,
#   'shadow_max_age': None,
#   'shadow_min_age': None,
#   'shadow_password': '$6$5H0QpwprRiJQR19Y$bXGOh7dIfOWpUb/Tuqr7yQVCqL3UkrJns9.7msfvMg4ZO/PsFC5Tbt32PXAw9qRFEBs1254aLimFeNM8YsYOv.',
#   'shadow_reserved': None,
#   'shadow_salt': '5H0QpwprRiJQR19Y',
#   'shadow_warning': None}, ...]

# %% Example

# Structure of `/etc/passwd` file [1]:
#
# | Field    | Type | Description                       |
# |----------|------|-----------------------------------|
# | login    | str  | login name                        |
# | password | str  | optional encrypted password       |
# | uid      | int  | numerical user ID                 |
# | gid      | int  | numerical group ID                |
# | comment  | str  | user name or comment field        |
# | home     | str  | user home directory               |
# | shell    | str  | optional user command interpreter |
#
# If the password field is a lower-case 'x', then the encrypted
# password is actually stored in the `/etc/shadow` file instead
#
# Example:
# - alice:x:1000:1000:Alice:/home/alice:/bin/bash
# - [login]:[password]:[uid]:[gid]:[comment]:[home]:[shell]

# Structure of `/etc/shadow` file [2]:
#
# | Field       | Type         | Description                                                     |
# |-------------|--------------|-----------------------------------------------------------------|
# | login       | str          | login name, matching `/etc/passwd`                              |
# | password    | str          | encrypted password (see below for more details)                 |
# | last_change | date or None | days since 1970-01-01 when the password was changed             |
# | min_age     | int or None  | days before which password may not be changed                   |
# | max_age     | int or None  | days after which password must be changed                       |
# | warning     | int or None  | days before `max_age` to warn the user to change their password |
# | inactive    | int or None  | days after password expires that account is disabled            |
# | expiration  | date or None | days since 1970-01-01 when account will be disabled             |
# | reserved    | str or None  | this field is reserved for future use                           |
#
# Example:
# - alice:$6$wXtY9ZoG$MzaxvKfj3Z8F9G8wKz7LU0:18736:0:99999:7:::
# - [login]:[password]:[last_change]:[min_age]:[max_age]:[warning]:[inactive]:[expiration]:[reserved]
#
# Password field (split by `$`):
# - modifier
# - algorithm
# - salt
# - password hash
#
# Password algorithms [3]:
# - 1 - md5crypt
# - 2a - bcrypt (Blowfish)
# - 2b - bcrypt (Blowfish)
# - 2x - bcrypt (Blowfish)
# - 2y - bcrypt (Blowfish)
# - 3 - NTHASH
# - 5 - sha256crypt
# - 6 - sha512crypt
# - 7 - scrypt
# - y - yescrypt
# - 8 - PBKDF2 with different implementations
# - gy - gost-yescrypt
# - md5 - Solaris MD5
# - sha1 - PBKDF1 with SHA-1
# - none of the above - DES
#
# Password modifiers:
#
# | Modifier | Locked | Algorithm | Salt | Hash | Comment                                                   |
# |----------|--------|-----------|------|------|-----------------------------------------------------------|
# | ' '      | False  | None      | None | None | password not required to log in                           |
# | '*'      | True   | None      | None | None | password authentication disabled, can use `su` or SSH-key |
# | '!!'     | True   | None      | None | None | account inactivated by admin                              |
# | '!'      | True   | yes       | yes  | yes  | password authentication disabled, can use `su` or SSH-key |
# | ''       | False  | yes       | yes  | yes  | normal account                                            |
#
#
# Example 1:
# - $6$abcde$A1b2c3d4E5
# - [modifier]$[algorithm]$[salt]$[hash]
#
# - modifier: None
# - locked: False
# - algorithm: sha512crypt
# - salt: abcde
# - hash: A1b2c3d4E5
#
# Example 2:
# - !$1$abcde$A1b2c3d4E5
# - [modifier]$[algorithm]$[salt]$[hash]
#
# - modifier: !
# - locked: True
# - algorithm: md5crypt
# - salt: abcde
# - hash: A1b2c3d4E5
#
# Example 3:
# - *
# - [modifier]$[algorithm]$[salt]$[hash]
#
# - modifier: None
# - locked: True
# - algorithm: None
# - salt: None
# - hash: None

# Structure of `/etc/group` file:
#
# | Field    | Type | Description                                   |
# |----------|------|-----------------------------------------------|
# | name     | str  | group name                                    |
# | password | str  | 'x' indicates that shadow passwords are used  |
# | gid      | int  | group id                                      |
# | members  | list | comma-separated logins matching `/etc/passwd` |
#
# Example:
# - users::1001:alice,bob,carol,dave,eve
# - [name]:[password]:[gid]:[members]

# %% Hints
# - `len()`
# - `open().read()`
# - `str.splitlines()`
# - `str.strip()`
# - `str.startswith()`
# - `str.split()`
# - `str.isspace()`
# - `list.append()`
# - `datetime.date(1970, 1, 1)`
# - `datetime.timedelta(days=...)`
# - `dict.get()`
# - `dict.update()`
# - `dict1 | dict2`
# - `... if ... else ...`
# - `passwd['mwatney'] = {...}`
# - `shadow['mwatney'] = {...}`
# - `groups['mwatney'] = {...}`
# - `result['mwatney'] = dict1 | dict2 | dict3`

# %% Doctests
"""
>>> import sys; sys.tracebacklimit = 0

>>> assert sys.version_info >= (3, 9), \
'Python has an is invalid version; expected: `3.9` or newer.'

>>> from pprint import pprint
>>> pprint(result)
[{'groups_names': ['users', 'staff'],
  'passwd_comment': 'Alice',
  'passwd_gid': 1000,
  'passwd_home': '/home/alice',
  'passwd_login': 'alice',
  'passwd_password': 'x',
  'passwd_shell': '/bin/bash',
  'passwd_uid': 1000,
  'shadow_algorithm': 'sha512crypt',
  'shadow_expiration': None,
  'shadow_hash': 'bXGOh7dIfOWpUb/Tuqr7yQVCqL3UkrJns9.7msfvMg4ZO/PsFC5Tbt32PXAw9qRFEBs1254aLimFeNM8YsYOv.',
  'shadow_inactive': None,
  'shadow_last_change': datetime.date(2000, 1, 1),
  'shadow_locked': False,
  'shadow_max_age': 99999,
  'shadow_min_age': 0,
  'shadow_password': '$6$5H0QpwprRiJQR19Y$bXGOh7dIfOWpUb/Tuqr7yQVCqL3UkrJns9.7msfvMg4ZO/PsFC5Tbt32PXAw9qRFEBs1254aLimFeNM8YsYOv.',
  'shadow_reserved': None,
  'shadow_salt': '5H0QpwprRiJQR19Y',
  'shadow_warning': 7},
 {'groups_names': ['users', 'staff'],
  'passwd_comment': 'Bob',
  'passwd_gid': 1001,
  'passwd_home': '/home/bob',
  'passwd_login': 'bob',
  'passwd_password': 'x',
  'passwd_shell': '/bin/bash',
  'passwd_uid': 1001,
  'shadow_algorithm': 'sha512crypt',
  'shadow_expiration': None,
  'shadow_hash': 'tgfvvFWJJ5FKmoXiP5rXWOjwoEBOEoAuBi3EphRbJqqjWYvhEM2wa67L9XgQ7W591FxUNklkDIQsk4kijuhE50',
  'shadow_inactive': None,
  'shadow_last_change': datetime.date(2000, 1, 1),
  'shadow_locked': False,
  'shadow_max_age': 99999,
  'shadow_min_age': 0,
  'shadow_password': '$6$P9zn0KwR$tgfvvFWJJ5FKmoXiP5rXWOjwoEBOEoAuBi3EphRbJqqjWYvhEM2wa67L9XgQ7W591FxUNklkDIQsk4kijuhE50',
  'shadow_reserved': None,
  'shadow_salt': 'P9zn0KwR',
  'shadow_warning': 7},
 {'groups_names': ['users'],
  'passwd_comment': 'Carol',
  'passwd_gid': 1002,
  'passwd_home': '/home/carol',
  'passwd_login': 'carol',
  'passwd_password': 'x',
  'passwd_shell': '/bin/bash',
  'passwd_uid': 1002,
  'shadow_algorithm': 'md5crypt',
  'shadow_expiration': None,
  'shadow_hash': 'SWlkjRWexrXYgc98F.',
  'shadow_inactive': None,
  'shadow_last_change': datetime.date(2000, 1, 1),
  'shadow_locked': True,
  'shadow_max_age': 99999,
  'shadow_min_age': 0,
  'shadow_password': '!$1$.QKDPc5E$SWlkjRWexrXYgc98F.',
  'shadow_reserved': None,
  'shadow_salt': '.QKDPc5E',
  'shadow_warning': 7},
 {'groups_names': ['users'],
  'passwd_comment': 'Dave',
  'passwd_gid': 1003,
  'passwd_home': '/home/dave',
  'passwd_login': 'dave',
  'passwd_password': 'x',
  'passwd_shell': '/bin/bash',
  'passwd_uid': 1003,
  'shadow_algorithm': 'sha512crypt',
  'shadow_expiration': None,
  'shadow_hash': 'MxaKfj3Z8F9G8wKz7LU0',
  'shadow_inactive': None,
  'shadow_last_change': datetime.date(2000, 1, 1),
  'shadow_locked': True,
  'shadow_max_age': 99999,
  'shadow_min_age': 0,
  'shadow_password': '!$6$wXtY9ZoG$MxaKfj3Z8F9G8wKz7LU0',
  'shadow_reserved': None,
  'shadow_salt': 'wXtY9ZoG',
  'shadow_warning': 7},
 {'groups_names': ['users', 'staff', 'admins'],
  'passwd_comment': 'Eve',
  'passwd_gid': 1004,
  'passwd_home': '/home/eve',
  'passwd_login': 'eve',
  'passwd_password': 'x',
  'passwd_shell': '/bin/bash',
  'passwd_uid': 1004,
  'shadow_algorithm': None,
  'shadow_expiration': datetime.date(2000, 2, 1),
  'shadow_hash': None,
  'shadow_inactive': 30,
  'shadow_last_change': datetime.date(2000, 1, 1),
  'shadow_locked': True,
  'shadow_max_age': 90,
  'shadow_min_age': 0,
  'shadow_password': '!!',
  'shadow_reserved': None,
  'shadow_salt': None,
  'shadow_warning': 5},
 {'groups_names': [],
  'passwd_comment': 'Mallory',
  'passwd_gid': 1005,
  'passwd_home': '/home/mallory',
  'passwd_login': 'mallory',
  'passwd_password': 'x',
  'passwd_shell': '/bin/bash',
  'passwd_uid': 1005,
  'shadow_algorithm': None,
  'shadow_expiration': None,
  'shadow_hash': None,
  'shadow_inactive': None,
  'shadow_last_change': None,
  'shadow_locked': False,
  'shadow_max_age': None,
  'shadow_min_age': None,
  'shadow_password': ' ',
  'shadow_reserved': None,
  'shadow_salt': None,
  'shadow_warning': None}]

  >>> from os import remove
  >>> remove(FILE_GROUPS)
  >>> remove(FILE_SHADOW)
  >>> remove(FILE_PASSWD)
"""

# %% Run
# - PyCharm: right-click in the editor and `Run Doctest in ...`
# - PyCharm: keyboard shortcut `Control + Shift + F10`
# - Terminal: `python -m doctest -f -v myfile.py`

# %% Imports
from datetime import date, timedelta

# %% Types
result: list[dict[str, str|int|bool]]

# %% Data
FILE_PASSWD = 'etc_passwd.txt'
FILE_SHADOW = 'etc_shadow.txt'
FILE_GROUPS = 'etc_groups.txt'

ALGORITHMS = {
    '1': 'md5crypt',
    '2a': 'bcrypt (Blowfish)',
    '2b': 'bcrypt (Blowfish)',
    '2x': 'bcrypt (Blowfish)',
    '2y': 'bcrypt (Blowfish)',
    '3': 'NTHASH',
    '5': 'sha256crypt',
    '6': 'sha512crypt',
    '7': 'scrypt',
    'y': 'yescrypt',
    '8': 'PBKDF2',
    'gy': 'gost-yescrypt',
    'md5': 'Solaris MD5',
    'sha1': 'PBKDF1 with SHA-1',
    '': 'DES',
}


with open(FILE_PASSWD, mode='wt', encoding='utf-8') as file:
    file.write("""# File: /etc/passwd
root:x:0:0:root:/root:/bin/bash
daemon:x:1:1:daemon:/usr/sbin:/usr/sbin/nologin
bin:x:2:2:bin:/bin:/usr/sbin/nologin
sys:x:3:3:sys:/dev:/usr/sbin/nologin
alice:x:1000:1000:Alice:/home/alice:/bin/bash
bob:x:1001:1001:Bob:/home/bob:/bin/bash
carol:x:1002:1002:Carol:/home/carol:/bin/bash
dave:x:1003:1003:Dave:/home/dave:/bin/bash
eve:x:1004:1004:Eve:/home/eve:/bin/bash
mallory:x:1005:1005:Mallory:/home/mallory:/bin/bash
""")


with open(FILE_SHADOW, mode='wt', encoding='utf-8') as file:
    file.write("""# File: /etc/shadow
root:*:10957:0:99999:7:::
daemon:*:10957:0:99999:7:::
bin:*:10957:0:99999:7:::
sys:*:10957:0:99999:7:::
alice:$6$5H0QpwprRiJQR19Y$bXGOh7dIfOWpUb/Tuqr7yQVCqL3UkrJns9.7msfvMg4ZO/PsFC5Tbt32PXAw9qRFEBs1254aLimFeNM8YsYOv.:10957:0:99999:7:::
bob:$6$P9zn0KwR$tgfvvFWJJ5FKmoXiP5rXWOjwoEBOEoAuBi3EphRbJqqjWYvhEM2wa67L9XgQ7W591FxUNklkDIQsk4kijuhE50:10957:0:99999:7:::
carol:!$1$.QKDPc5E$SWlkjRWexrXYgc98F.:10957:0:99999:7:::
dave:!$6$wXtY9ZoG$MxaKfj3Z8F9G8wKz7LU0:10957:0:99999:7:::
eve:!!:10957:0:90:5:30:10988:
mallory: :::::::
""")


with open(FILE_GROUPS, mode='wt', encoding='utf-8') as file:
    file.write("""# File: /etc/group
root::0:root
daemon::1:root,daemon
bin::2:root,bin,daemon
sys::3:root
sudo::4:
users::1001:alice,bob,carol,dave,eve
staff::1002:alice,bob,eve
admins::1003:eve
""")

# %% Result
result = ...