# use direnv layout python instead
# not... pip install --upgrade pip; setuptools virtualenv

# list and dict comprehension. See below
[(x) for x in [1,2,3] if x != 2]
{x: x**2 for x in (2, 4, 6)}

print('.', end='')

from devtools import debug
pip install rich

if __name__ == "__main__":
	main()

	
pip install -U pip
pip install -r requirements.txt
pip install -r https://github...

python -c "import boto3; print(boto3.__version__)"


print(bin(42))       # '0b101010'                                                 
print(f"{42:b}")     # '101010'                                                   
print(f"{42:08b}")   # '00101010' (zero-padded to 8 bits)
print(f"{0xffff:b}")
print(f"{2**120:b}")

• Sample main with arg processing

  #!/usr/bin/env python
  import argparse
  import os
  from dotenv import dotenv_values
  
  class MyClass:
      def __init__(self, config1):
          self.config = config1
          pass
  
      def main(self, params):
  				pass
  
  if __name__ == "__main__":
      config = {
          **dotenv_values(".env"),  # load shared development variables
          **os.environ,  # override loaded values with environment variables
      }
      parser = argparse.ArgumentParser(description="create_tenant")
      parser.add_argument('--name', type=str, help='Tenant Name', required=True)
      parser.add_argument('--domain', type=str, help='Tenant Domain', required=True)
      args = parser.parse_args()
      MyClass(config).main(args)

• Docs

  • See My Dev Notes on Python

  • Types

  • Standard Lib

  • Learning

    • Python3 Tutorial

    • TutorialsPoint

    • W3Schools Python

  • docs.python.org

    • Python Docs

    • Std Lib

    • stdtypes

    • reference

    • asyncio — async/await

    • builtin

    • datetime

    • The Pyhton Data Model

    • Descriptor Howto Guide

    • HandlingExceptions

    • Compound statements

    • Logging Cookbook

    • defaultdict - search how to use

    • Time

      • Zoneinfo

      • Yesterday

      from datetime import date
      from datetime import timedelta
      today = date.today()
      yesterday = today - timedelta(days = 1)

    • List Comprehension

      [(x, y) for x in [1,2,3] for y in [3,1,4] if x != y]
      
      # can be expanded
      [
      	(x, y) 
      	for x in [1,2,3] 
      	for y in [3,1,4] 
      	if x != y
      ]

      • Dense Example

        return [
                name
                for _, name, is_pkg in pkgutil.iter_modules([command_dir])
                if not is_pkg and not name.startswith("_")
            ]

    • Dict Comprehension

      • https://www.datacamp.com/tutorial/python-dictionary-comprehension

      {x: x**2 for x in (2, 4, 6)}
      {2: 4, 4: 16, 6: 36}
      
      dict1 = {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5}
      # Double each value in the dictionary
      double_dict1 = {k:v*2 for (k,v) in dict1.items()}
      print(double_dict1)
      
      for i, v in enumerate(['tic', 'tac', 'toe']):
           print(i, v)

    • Print Formatting

      • f-strings

      • string and byte literals

    • Dict to Object

      class Struct:
          def __init__(self, **entries):
              self.__dict__.update(entries)
      args = {'a': 1, 'b': 2}
      s = Struct(**args)
      s # <__main__.Struct instance at 0x01D6A738>
      s.a # 1
      
      # json
      class obj_hook:
          def __init__(self, dict1):
              self.__dict__.update(dict1)
      json.loads(user_json, object_hook=obj_hook)

• Libs

  • https://github.com/vinta/awesome-python

  • https://github.com/JetBrains/awesome-pycharm

  • Network

    • libftp

      • https://linuxconfig.org/how-to-connect-to-an-ftp-server-using-python

    • gevent.org - User greenlet for high-level API

  • Shell execution

    • Fabric - remote

    • Invoke - local

    • Click - CLI util

    • typer.tiangolo.com — CLI util

    • shtab (tab completion) — not great

      • https://docs.iterative.ai/shtab/

    • https://github.com/tartley/colorama

  • Databases

    • https://github.com/redis/redis-py

  • Auth

    • osohq.com — Auth

  • Parsing

    • Sly Lex-Yacc

  • Templating

    • Jinja — fast, expressive, extensible templating engine

  • Monitoring

    • Sentry

  • Text Formatting

    • https://github.com/Textualize/rich - rich text formatting

    • textwrap dedent

  • Email

    • See boto3 section

    • https://github.com/alex-oleshkevich/mailers

    • Django Sending email

    Simple Email Service (SES)

  • Configuration/ENV

    • https://github.com/alex-oleshkevich/environs

  • Spreadsheets

    • openpyxl code - docs

  • Database

    • SQLAlchemy

    • GiNO - async

    • asyncpg - Async Postgres

    • macOS setup

    • pyodbc
      • https://github.com/mkleehammer/pyodbc
      • from macos

    • Read DDL from db

      • GetMetadataWithPython

    • SQLAlchemy

      • https://github.com/zzzeek/sqlalchemy
        • https://docs.sqlalchemy.org

      • Logging

      • pyodbc.py

  • Web Scraping

    • Scrapy

      • https://docs.scrapy.org/en/latest/

      • https://thepythonscrapyplaybook.com/

      • https://thepythonscrapyplaybook.com/freecodecamp-beginner-course/

      • https://github.com/orgs/python-scrapy-playbook/repositories

      • https://github.com/scrapy/scrapyd

      • https://github.com/my8100/scrapydweb

      • https://scrapeops.io/monitoring-scheduling/

      • https://www.zyte.com/scrapy-cloud/

      • https://scrapeops.io/proxy-providers/comparison/free-proxy-providers

      • https://scrapeops.io/proxy-api-aggregator/

      • https://smartproxy.com/deals/proxyservers/ips

  • Security

    • https://pyup.io/ — secure third-party modules

  • JSON

    • https://github.com/stefankoegl/python-json-patch

    import json
    
    try:
        with open('data.json', 'r') as file:
            data = json.load(file)
        print("File data =", data)   
    except FileNotFoundError:
        print("Error: The file 'data.json' was not found.")
    except json.JSONDecodeError:
        print("Error: Failed to decode JSON from the file.")
    

  • YAML

    • GG Tutorial on YAML

    • PyYAML

    import yaml
    data = {
        'name': 'Amit',
        'age': 25,
        'city': 'Mumbai',
        'skills': ['Python', 'Data Analysis', 'Machine Learning']
    }
    
    with open('data.yaml', 'w') as file:
        yaml.dump(data, file)
    
    with open('data.yaml', 'r') as file:
        loaded_data = yaml.safe_load(file)

  • GraphQL

    • https://graphene-python.org/

  • Docker

    • https://github.com/google/containerregistry - interact with docker registry

• Notebooks

  • Google Colab

  • nteract.io

    • https://github.com/nteract/desktop

    pip install runtimed
    pip install nteract

  • Jupyter

    • docs

    • installation

    pip3 install --upgrade pip
    pip3 install jupyter
    jupyter notebook
    jupyter notebook notebook.ipynb

    • Try Online - C++, Ruby, …

      • https://github.com/jupyter-xeus/xeus

      • https://xeus.readthedocs.io/en/latest/

      • https://github.com/compiler-research/xeus-cpp

      • https://xeus-cpp.readthedocs.io/en/latest/

    • https://github.com/jupyterhub/jupyterhub

      • custom-server.html

      • nbgitpuller.html

      • troubleshooting Traefik Logs

    • MyBinder

      • Users Guide

    • Links

      • Jupyter-kernels

      • DevOps Intel at Netflix

      • ProjectJupyter on X.com

• Data Science

  • CodeCut - stay sharp in data science one tip at a time

  • docs.pydantic.dev - Data

  • pydantic — Data validation and settings management using python type annotations

  • https://thispointer.com/ - Pandas, Numpy, …

  • https://github.com/drgrib/dotmap

  • Numba

  • NumPy

    • https://numpy.org/doc/stable/index.html

  • Pandas

    • Pandas_Cheat_Sheet.pdf

    • Pandas: Cheatsheet - Data Science

    • DataFrame

    • Auto set Date types

    • NaN is a Float

      • Pandas integer_na

      • See NA and missing data handling

  • Scikit

    • Learn scikit

• Machine Learning

  • PyTorch

    • https://www.learnpytorch.io/

    • https://www.freecodecamp.org/news/learn-pytorch-for-deep-learning-in-day/

  • Machine Learning with Python

    https://scikit-learn.org/stable/install.html

    pip install fastapi[all]
    pip install -U scikit-learn
    uvicorn main:app --reload
    
    npm install -g angular-cli
    ng serve

• Tools / Shells

  • PyCharm - jetbrains IDE

  • Spyder IDE

  • Astral - next-gen tooling
    • Github Astral

  • bpython

  • ipython

  • https://github.com/microsoft/vscode-python
    • Extension

  • Automation

    • https://github.com/ansible/ansible

    • https://github.com/canonical/cloud-init

    • https://github.com/sebastien/cuisine

    • https://github.com/fabric/fabric

  • Virtual Env

    • direnv layout python is the best

      # .envrc
      layout python

    • Check conda

    • https://github.com/pypa/pipx
      • PIPx

    • venv

• Testing

  • Pytest freecodecamp

  • ChaosToolkit - https://github.com/chaostoolkit/chaostoolkit

• Training

  • https://pythonprogramminglanguage.com/

  • VsCode Getting Started with Python

  • Stanford - Python Guide

  • Harvard CS50

  • Good content on YouTube
    • Gensim

  • FullStackPython JSON

  • Python For Everyone

  • note.nkmk.me Python

  • Check if number is an int

  • Learning / Lookup

    • what is: flake8, mypy,

    • Check for int or floats

    • https://realpython.com/

    • W3 Tutorial

    • Geeks for geeks Python

    • Coderwall Python

    • Python in one picture

      • https://github.com/coodict/python3-in-one-pic

      • http://coodict.github.io/python3-in-one-pic/

      • Python3 in one picture

      • py3 in one notebook

• Module Loading Help

  • Attemped relative import with no known parent package

  • List modules and directories

    python -c "import sys; print('\n'.join(sys.path))"
    python -c 'import sys; print ("\n".join(list(sorted(list(map(lambda x: str(x),sys.modules))))))'

• Documenting

  • reStructuredText
    • reStructured Text Ref
    • Demo Page

• Snipets

  import os
  import os.path
  try:
    os.getcwd()
  except OSError:
    try:
      os.chdir(os.path.expanduser('~'))
    except OSError:
      os.chdir('/')
  
  # read file
  try:
      with open('data.json', 'r') as file:
          data = file.read()
      print("File data =", data)   
  except FileNotFoundError:
      print("Error: The file 'data.json' was not found.")
      
  # path list
  print "\n".join(sys.path)
  
  # get ip
  hostname -I
  python -c "import socket; s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); s.connect(('8.8.8.8', 80)); print(s.getsockname()[0]); s.close()"