-
Notifications
You must be signed in to change notification settings - Fork 12
/
mario.py
143 lines (116 loc) · 5.69 KB
/
mario.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
#!/usr/bin/env python
"""Mario Gym Adventure!
python mario.py ppaquette/meta-SuperMarioBros-v0 \
-a DQFDAgent -c mario_agent.json \
-ld ./demos/ -s ./agents/ -m ./monitors/ -mv 1000 -D
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import logging
import os
import demo
from model import mario_net
from ppaquette_gym_super_mario import wrappers
from tensorforce import Configuration, TensorForceError
from tensorforce.agents import agents
from tensorforce.contrib.openai_gym import OpenAIGym
from tensorforce.execution import Runner
def main():
parser = argparse.ArgumentParser()
parser.add_argument('gym_id', help="ID of the gym environment, i.e. ppaquette/SuperMarioBros-1-1-v0")
parser.add_argument('-a', '--agent', help='Agent')
parser.add_argument('-c', '--agent-config', help="Agent configuration file")
parser.add_argument('-e', '--episodes', type=int, default=50000, help="Number of episodes")
parser.add_argument('-t', '--max-timesteps', type=int, default=100000, help="Maximum number of timesteps per episode")
parser.add_argument('-m', '--monitor', help="Save results to this directory")
parser.add_argument('-ms', '--monitor-safe', action='store_true', default=False, help="Do not overwrite previous results")
parser.add_argument('-mv', '--monitor-video', type=int, default=0, help="Save video every x steps (0 = disabled)")
parser.add_argument('-s', '--save', help="Save agent to this dir")
parser.add_argument('-se', '--save-episodes', type=int, default=100, help="Save agent every x episodes")
parser.add_argument('-l', '--load', help="Load agent from this dir")
parser.add_argument('-D', '--debug', action='store_true', default=False, help="Show debug outputs")
parser.add_argument('-ld', '--load-demo', required=True, help="Load demos from this dir")
parser.add_argument('-pt', '--pretrain', action='store_true', default=False, help="Pretrain agent on demos")
parser.add_argument('-ul', '--use_lstm', action='store_true', default=False, help="Use LSTM model")
parser.add_argument('-ls', '--lstm_size', type=int, default=256, help="LSTM size")
args = parser.parse_args()
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
environment = OpenAIGym(args.gym_id,
monitor=args.monitor,
monitor_safe=args.monitor_safe,
monitor_video=args.monitor_video)
mode_wrapper = wrappers.SetPlayingMode('algo')
ac_wrapper = wrappers.ToDiscrete()
environment.gym = mode_wrapper(ac_wrapper(environment.gym))
if args.agent_config:
agent_config = Configuration.from_json(args.agent_config)
else:
agent_config = Configuration()
logger.info("No agent configuration provided.")
agent_config.default(dict(states=environment.states,
actions=environment.actions,
network=mario_net(name='mario',
lstm_size=args.lstm_size,
actions=environment.actions['num_actions'],
use_lstm=args.use_lstm)))
agent = agents[args.agent](config=agent_config)
if args.load:
load_dir = os.path.dirname(args.load)
if not os.path.isdir(load_dir):
raise OSError("Could not load agent from {}: No such directory.".format(load_dir))
logger.info("-" * 16)
agent.load_model(args.load)
logger.info("Loaded {}".format(agent))
if args.debug:
logger.info("-" * 16)
logger.info("Configuration:")
logger.info(agent_config)
if args.save:
save_dir = os.path.dirname(args.save)
if not os.path.isdir(save_dir):
try:
os.mkdir(save_dir, 0o755)
except OSError:
raise OSError("Cannot save agent to dir {} ()".format(save_dir))
try:
if args.load_demo:
logger.info("-" * 16)
logger.info("Loading demos")
demos = demo.load(args.load_demo)
logger.info("Importing demos")
agent.import_demonstrations(demos)
if args.pretrain:
logger.info("-" * 16)
logger.info("Pretraining {} steps".format(len(demos)))
agent.pretrain(steps=len(demos))
runner = Runner(
agent=agent,
environment=environment,
repeat_actions=1,
save_path=args.save,
save_episodes=args.save_episodes
)
report_episodes = args.episodes // 1000
if args.debug:
report_episodes = 1
def episode_finished(r):
if r.episode % report_episodes == 0:
logger.info("Finished episode {ep} after {ts} timesteps".format(ep=r.episode, ts=r.timestep))
logger.info("Episode reward: {}".format(r.episode_rewards[-1]))
logger.info("Average of last 500 rewards: {}".format(sum(r.episode_rewards[-500:]) / 500))
logger.info("Average of last 100 rewards: {}".format(sum(r.episode_rewards[-100:]) / 100))
return True
logger.info("Starting {agent} for Environment '{env}'".format(agent=agent, env=environment))
runner.run(args.episodes, args.max_timesteps, episode_finished=episode_finished)
logger.info("Learning finished. Total episodes: {ep}".format(ep=runner.episode))
except (KeyboardInterrupt):
agent.save_model(args.save)
pass
if args.monitor:
environment.gym.monitor.close()
environment.close()
if __name__ == '__main__':
main()