Simon Shi的小站

人工智能，机器学习， 强化学习，大模型，自动驾驶

0%

python APIs

Posted on Edited on In ,

python 备忘录

[TOC]

简单API

FUN: 重命名目录下文件

  • filter
  • lambda
1
2
3
4
5
6
def format_filename(dir):
    fs = os.listdir(dir)
    for f in fs:
        new_n = ''.join(filter(lambda ch: ch not in '# (){}#', f))
        print(f, ' ----> ', new_n)
        os.rename(dir+f, dir+new_n)

traceback

1
2
3
4
5
6
def log_error(e):
    print('err:', e)
    print('err:', request.form)
    print('err file:', e.__traceback__.tb_frame.f_globals["__file__"])  # 发生异常所在的文件
    print('err file line:', e.__traceback__.tb_lineno)  # 发生异常所在的行数
    print('err ', traceback.print_exc())

Time

1
2
3
4
5
6
>>> time.ctime()
'Thu May 5 14:58:09 2011'
>>> time.ctime(time.time())
'Thu May 5 14:58:39 2011'
>>> time.ctime(1304579615)
'Thu May 5 15:13:35 2011'

os

1
os.mkdir()

os.path

os.path.basename()

os.path.dirname()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
path = '/home/User/Documents'
dirname = os.path.dirname(path)
print(dirname)


path = '/home/User/Documents/file.txt'
dirname = os.path.dirname(path)
print(dirname)


path = 'file.txt'
dirname = os.path.dirname(path)
print(dirname)

# 三个的结果：
>>>/home/User
>>>/home/User/Documents
>>>

expanduser

python的os.path 模块提供了一个expanduser函数，它可以将参数中开头部分的 ~ 或 ~user 替换为当前用户的home目录并返回，仅看定义难以理解，我在linux系统和winodws系统下分别实验它的功能。

在linux系统下，我的账号是kwsy，这个用户的home目录是/home/kwsy，下面的代码演示如何使用expanduser函数。

1
2
3
4
5
6
Python 3.7.0 (default, Jun 28 2018, 13:15:42)
[GCC 7.2.0] :: Anaconda, Inc. on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import os
>>> os.path.expanduser('~/.config/')
'/home/kwsy/.config/'

expandvars

Python中的方法用于扩展给定路径中的环境变量。它将给定路径中形式为$name$ 或${name}$的子字符串替换为环境变量name的值。

1
todo

random

1
random.randint(0, N) -> one of [0,1, ...,N]

str

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
text = 'hello world'

# 1. in
if 'w' in text:
    print('find w in text')

if 'hello' in text:
    print('find')

if 'hello' not in text:
    print(' not find ')


# 2.find func
if text.find('xxx') != -1:
     print('find position=', text.find('xxx'))

# 3 index func
try:
    position = text.index('not exits char')
except ValueError:
    print('not find ')

dict_2_obj

1、手动映射字段

1
2
3
4
5
6
class Person:
    def __init__(self, name, age):
        self.name = name
        self.age = age

person = Person(data['name'], data['age'])

2、自动映射object_hook

使用 json.loads() 的 object_hook 参数，自动将字典转换为对象‌

1
2
3
4
5
6
7
class JsonObject:
    def __init__(self, d):
        self.__dict__ = d

json_str = '{"name": "John", "age": 30}'
obj = json.loads(json_str, object_hook=JsonObject)
print(obj.name)  # 输出：John

3、动态属性拓展json to object

支持动态解析复杂JSON结构（如嵌套字典或列表）‌

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
import json
class obj(object):
    def __init__(self, dict_):
        self.__dict__.update(dict_)

def dict2obj(d):
    return json.loads(json.dumps(d), object_hook=obj)

d = {'a': 1, 'b': {'c': 2}, 'd': ['hi', {'foo': 'cjavapy'}]}
# 使用1
o = dict2obj(d)
print(o)
print(o.a)
print(o.b)
print(o.d)
print(o.d[1].foo)

进阶1 自定义+递归

1
2
3
4
5
6
7
8
9
10
11
12
13
14
class obj(object):
    def __init__(self, d):
        for a, b in d.items():
            if isinstance(b, (list, tuple)):
               setattr(self, a, [obj(x) if isinstance(x, dict) else x for x in b])
            else:
               setattr(self, a, obj(b) if isinstance(b, dict) else b)
d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "cjavapy"}]}
o = obj(d)
print(o)
print(o.a)
print(o.b)
print(o.d)
print(o.d[1].foo)

json

1
2
3
4
5
6
7
8
9
10
11
12
13
import json
class obj(object):
    def __init__(self, dict_):
        self.__dict__.update(dict_)
def dict2obj(d):
    return json.loads(json.dumps(d), object_hook=obj)
d = {'a': 1, 'b': {'c': 2}, 'd': ['hi', {'foo': 'cjavapy'}]}
o = dict2obj(d)
print(o)
print(o.a)
print(o.b)
print(o.d)
print(o.d[1].foo)

进阶2 namedtuple + 递归

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
from collections import namedtuple
class struct(object):
    def __new__(cls, data):
        if isinstance(data, dict):
            return namedtuple(
                'struct', data.keys()
            )(
                *(struct(val) for val in data.values())
            )
        elif isinstance(data, (tuple, list, set, frozenset)):
            return type(data)(struct(_) for _ in data)
        else:
            return data
d = {'a': 1, 'b': {'c': 2}, 'd': ['hi', {'foo': 'cjavapy'}]}
o = struct(d)
print(o)
print(o.a)
print(o.b)
print(o.d)
print(o.d[1].foo)

进阶3 自定义编码、解码器

通过继承 json.JSONEncoder 和 json.JSONDecoder 处理特殊数据类型（如日期、枚举)

1
2
3
4
5
6
7
class CustomEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, datetime):
            return obj.isoformat()
        return super().default(obj)

json_str = json.dumps(obj, cls=CustomEncoder)

进阶4 复杂递归

1
2
3
4
5
6
7
8
9
10
11
class NestedObject:
    def __init__(self, d):
        for key, value in d.items():
            if isinstance(value, dict):
                setattr(self, key, NestedObject(value))
            else:
                setattr(self, key, value)

json_str = '{"books": {"语文": "济南的冬天"}}'
obj = json.loads(json_str, object_hook=NestedObject)
print(obj.books.语文)  # 输出：济南的冬天

数据文件读写

json

1
2
3
4
5
6
7
json.dumps()
json.loads()
import json

# json.dumps()函数的使用，将字典转化为字符串
dict1 = {"age": "12"}
json_info = json.dumps(dict1)
1
2
3
4
5
6
7
# Writing JSON data
with open('data.json', 'w') as f:
    json.dump(data, f)

# Reading data back
with open('data.json', 'r') as f:
    data = json.load(f)

np

1
2
3
4
import numpy as np

a=np.array(a)
np.save(‘a.npy’,a)   # 保存为.npy格式
1
2
a=np.load(‘a.npy’)
a=a.tolist()

csv

1
2
3
4
import pandas as pd

readme = pd.read_csv(‘读我.txt’,sep=’:’,encoding=“utf-8”, engine=‘python’,header=None)
readme = np.array(readme)

高级API

组合函数 itertools.combinations()元素不可重复

1
2
3
itertools.combinations(iters,n)元组输出迭代器，需List显示，全组合直接append分三个元组。
combinations('ABCD', 2) --> AB AC AD BC BD CD
combinations(range(4), 3) --> 012 013 023 123

ref:组合blog

combinations

1
2
3
4
5
6
7
import itertools 
combinations2 = itertools.combinations(iterable, r)
for combo in combinations2: 
    print(combo)
(1, 2)
(1, 3)
(2, 3)

combinations_with_replacement() 元素可重复的组合

1
2
3
4
5
6
7
8
9
10
11
12
13
14
import itertools  

# 定义一个可迭代对象  
iterable = [1, 2, 3]  

# 设置组合长度为2  
r = 2  

# 生成可重复组合的迭代器  
combinations = itertools.combinations_with_replacement(iterable, r)  

# 打印每个组合  
for combo in combinations:  
    print(combo)

out

1
2
3
4
5
6
(1, 1)  
(1, 2)  
(1, 3)  
(2, 2)  
(2, 3)  
(3, 3)

@property的理解和使用

1
2
3
4
5
6
7
@property
def password(self):
    raise AttributeError('password is not a readable attribute')

@password.setter
def password(self, password):
    self.password_hash = generate_password_hash(password)
通常赋值函数
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
class Student(object):

    def get_score(self):
        return self._score

    def set_score(self, value):
        if not isinstance(value, int):
            raise ValueError('score must be an integer!')
        if value < 0 or value > 100:
            raise ValueError('score must between 0 ~ 100!')
        self._score = value


>>> s = Student()
>>> s.set_score(60) # ok!
>>> s.get_score()
60
>>> s.set_score(9999)
Traceback (most recent call last):
  ...
ValueError: score must between 0 ~ 100!
@property
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
但是为了方便,节省时间,我们不想写s.set_score(9999)啊,直接写s.score = 9999不是更快么,加了方法做限制不能让调用的时候变麻烦啊,@property快来帮忙….

class Student(object):

    @property
    def score(self):
        return self._score

    @score.setter
    def score(self,value):
        if not isinstance(value, int):
            raise ValueError('分数必须是整数才行呐')
        if value < 0 or value > 100:
            raise ValueError('分数必须0-100之间')
        self._score = value

看上面代码可知,把get方法变为属性只需要加上@property装饰器即可,此时@property本身又会创建另外一个装饰器@score.setter,负责把set方法变成给属性赋值,这么做完后,我们调用起来既可控又方便

>>> s = Student()
>>> s.score = 60 # OK，实际转化为s.set_score(60)
>>> s.score # OK，实际转化为s.get_score()
60
>>> s.score = 9999
Traceback (most recent call last):
  ...
ValueError: score must between 0 ~ 100!

MP

1
2
3
4
5
6
对于一般进程间共享数据来说，使用
multiprocessing.Manager().Value
和multiprocessing.Manager().list()
和multiprocessing.Manager().dict()即可。
Or:
Queue()

pyLint

1
2
3
4
5
6
7
8
9
10
11
12
# Install
pip install pylint
pylint --version

#使用,生成默认配置文件：
pylint --persistent=n --generate-rcfile > .pylintrc

# 检查单个文件：
pylint [options] m1.py

# 整个项目
pylint [options] project_path

re

去除中文标点符号

s = re.sub(r'[.,"\'-?:!;]', '', s)

去除中文

temp = re.sub('[\u4e00-\u9fa5]','',text)

去除英文

temp = re.sub('[a-zA-Z]','',text)

del数字

temp = re.sub('[\d]','',text) # [0-9]

del 空格

temp = re.sub('[\s]','',text) #temp = text.strip()

RUN

1
2
3
4
5
6
7
8
9
10
11
12
13
# 1 normal run
python -u main.py


# 2 subdir run
python demo/main.py
--修改main文件
import os, sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))


# 3 -m 运行
python -m model.main