1
2
3
4
5
6
7
8
9

import pickle 
from collections import defaultdict
import gym  
import numpy as np


Q = defaultdict(lambda: [0, 0, 0])

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

env = gym.make('MountainCar-v0')

def transform_state(state):
    """将 position, velocity 通过线性转换映射到 [0, 40] 范围内"""
    pos, v = state
    pos_low, v_low = env.observation_space.low
    pos_high, v_high = env.observation_space.high

    a = 40 * (pos - pos_low) / (pos_high - pos_low)
    b = 40 * (v - v_low) / (v_high - v_low)

    return int(a), int(b)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

lr, factor = 0.7, 0.95
episodes = 10000  
score_list = []  
for i in range(episodes):
    s = transform_state(env.reset())
    score = 0
    while True:
        a = np.argmax(Q[s])
        
        if np.random.random() > i * 3 / episodes:
            a = np.random.choice([0, 1, 2])
        
        next_s, reward, done, _ = env.step(a)
        next_s = transform_state(next_s)
        
        Q[s][a] = (1 - lr) * Q[s][a] + lr * (reward + factor * max(Q[next_s]))
        score += reward
        s = next_s
        if done:
            score_list.append(score)
            print('episode:', i, 'score:', score, 'max:', max(score_list))
            break
env.close()


with open('MountainCar-v0-q-learning.pickle', 'wb') as f:
    pickle.dump(dict(Q), f)
    print('model saved')

1
2
3
4
5
6
7
8
9

$ python q_learning.py
episode: 3080 score: -200.0 max: -200
episode: 3081 score: -200.0 max: -200
...
episode: 9996 score: -169.0 max: -119.0
episode: 9997 score: -141.0 max: -119.0
episode: 9998 score: -160.0 max: -119.0
episode: 9999 score: -161.0 max: -119.0
model saved

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27

import time
import pickle
import gym
import numpy as np


with open('MountainCar-v0-q-learning.pickle', 'rb') as f:
    Q = pickle.load(f)
    print('model loaded')

env = gym.make('MountainCar-v0')
s = env.reset()
score = 0
while True:
    env.render()
    time.sleep(0.01)
    
    s = transform_state(s)
    a = np.argmax(Q[s]) if s in Q else 0
    s, reward, done, _ = env.step(a)
    score += reward
    if done:
        print('score:', score)
        break
env.close()

1
2
3

$ python test_q_learning.py
model loaded
score: -151.0