Merge pull request #44 from MarcCote/enh_game_options

Adding `GameOptions`

Author: tavianator

scripts/tw-make

@@ -3,8 +3,10 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT license.
 
-
+import os
 import argparse
+from os.path import join as pjoin
+
 import numpy as np
 
 import textworld
@@ -14,8 +16,8 @@ def parse_args():
     general_parser = argparse.ArgumentParser(add_help=False)
 
     general_group = general_parser.add_argument_group('General settings')
-    general_group.add_argument("--output", default="./gen_games/", metavar="PATH",
-                               help="Output folder to save generated game files.")
+    general_group.add_argument("--output", default="./tw_games/", metavar="PATH",
+                               help="Path where to save the generated game.")
     general_group.add_argument('--seed', type=int)
     general_group.add_argument("--view", action="store_true",
                                help="Display the resulting game.")
@@ -64,18 +66,21 @@ if __name__ == "__main__":
 
     print("Global seed: {}".format(args.seed))
 
-    grammar_flags = {
-        "theme": args.theme,
-        "include_adj": args.include_adj,
-        "only_last_action": args.only_last_action,
-        "blend_instructions": args.blend_instructions,
-        "blend_descriptions": args.blend_descriptions,
-        "ambiguous_instructions": args.ambiguous_instructions,
-    }
+    options = textworld.GameOptions()
+    options.grammar.theme = args.theme
+    options.grammar.include_adj = args.include_adj
+    options.grammar.only_last_action = args.only_last_action
+    options.grammar.blend_instructions = args.blend_instructions
+    options.grammar.blend_descriptions = args.blend_descriptions
+    options.grammar.ambiguous_instructions = args.ambiguous_instructions
 
     if args.subcommand == "custom":
-        game_file, game = textworld.make(args.world_size, args.nb_objects, args.quest_length, args.quest_breadth, grammar_flags,
-                                         seed=args.seed, games_dir=args.output)
+        options.nb_rooms = args.world_size
+        options.nb_objects = args.nb_objects
+        options.quest_length = args.quest_length
+        options.quest_breadth = args.quest_breadth
+        options.seeds = args.seed
+        game_file, game = textworld.make(options, args.output)
 
     elif args.subcommand == "challenge":
         _, challenge, level = args.challenge.split("-")
@@ -84,8 +89,8 @@ if __name__ == "__main__":
 
         level = int(level.lstrip("level"))
         make_game = textworld.challenges.CHALLENGES[challenge]
-        game = make_game(level=level, grammar_flags=grammar_flags, seeds=args.seed)
-        game_file = textworld.generator.compile_game(game, games_folder=args.output)
+        game = make_game(level, options)
+        game_file = textworld.generator.compile_game(game, args.output)
 
     print("Game generated: {}".format(game_file))
     if args.verbose:

scripts_dev/benchmark_framework.py

@@ -1,28 +1,40 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT license.
 
-
 import time
 import argparse
+from os.path import join as pjoin
+
+import numpy as np
 
 import textworld
 from textworld import g_rng
 from textworld.generator import World
 
+from textworld.generator.game import GameOptions
+
 
 def generate_never_ending_game(args):
     g_rng.set_seed(args.seed)
+
     msg = "--max-steps {} --nb-objects {} --nb-rooms {} --quest-length {} --quest-breadth {} --seed {}"
     print(msg.format(args.max_steps, args.nb_objects, args.nb_rooms, args.quest_length, args.quest_breadth, g_rng.seed))
     print("Generating game...")
 
-    grammar_flags = {}
-    game = textworld.generator.make_game(args.nb_rooms, args.nb_objects, args.quest_length, args.quest_breadth, grammar_flags)
+    options = GameOptions()
+    options.seeds = g_rng.seed
+    options.nb_rooms = args.nb_rooms
+    options.nb_objects = args.nb_objects
+    options.quest_length = args.quest_length
+    options.quest_breadth = args.quest_breadth
+
+    game = textworld.generator.make_game(options)
     if args.no_quest:
         game.quests = []
 
     game_name = "neverending"
-    game_file = textworld.generator.compile_game(game, game_name, force_recompile=True, games_folder=args.output)
+    path = pjoin(args.output, game_name + ".ulx")
+    game_file = textworld.generator.compile_game(game, path, force_recompile=True)
     return game_file
 
 
@@ -33,7 +45,7 @@ def benchmark(game_file, args):
     if args.mode == "random":
         agent = textworld.agents.NaiveAgent()
     elif args.mode == "random-cmd":
-        agent = textworld.agents.RandomCommandAgent()
+        agent = textworld.agents.RandomCommandAgent(seed=args.agent_seed)
     elif args.mode == "walkthrough":
         agent = textworld.agents.WalkthroughAgent()
 
@@ -52,14 +64,13 @@ def benchmark(game_file, args):
 
     reward = 0
     done = False
-    print("Benchmarking {} using ...".format(game_file, env.__class__.__name__))
     start_time = time.time()
     for _ in range(args.max_steps):
         command = agent.act(game_state, reward, done)
         game_state, reward, done = env.step(command)
 
         if done:
-            print("Win! Reset.")
+            #print("Win! Reset.")
             env.reset()
             done = False
 
@@ -69,6 +80,7 @@ def benchmark(game_file, args):
     duration = time.time() - start_time
     speed = args.max_steps / duration
     print("Done {:,} steps in {:.2f} secs ({:,.1f} steps/sec)".format(args.max_steps, duration, speed))
+    return speed
 
 
 def parse_args():
@@ -90,11 +102,21 @@ def parse_args():
     parser.add_argument("--compute_intermediate_reward", action="store_true")
     parser.add_argument("--activate_state_tracking", action="store_true")
     parser.add_argument("--seed", type=int)
+    parser.add_argument("--agent-seed", type=int, default=2018)
     parser.add_argument("-v", "--verbose", action="store_true")
     return parser.parse_args()
 
 
 if __name__ == "__main__":
     args = parse_args()
     game_file = generate_never_ending_game(args)
-    benchmark(game_file, args)
+
+
+    speeds = []
+    for _ in range(10):
+        speed = benchmark(game_file, args)
+        speeds.append(speed)
+        args.agent_seed = args.agent_seed + 1
+
+    print("-----\nAverage: {:,.1f} steps/sec".format(np.mean(speeds)))
+

tests/test_make_game.py

@@ -11,21 +11,34 @@ def test_making_game_with_names_to_exclude():
     g_rng.set_seed(42)
 
     with make_temp_directory(prefix="test_render_wrapper") as tmpdir:
-        game_file1, game1 = textworld.make(2, 20, 3, 3, {"names_to_exclude": []},
-                                           seed=123, games_dir=tmpdir)
-
+        options = textworld.GameOptions()
+        options.nb_rooms = 2
+        options.nb_objects = 20
+        options.quest_length = 3
+        options.quest_breadth = 3
+        options.seeds = 123
+        game_file1, game1 = textworld.make(options, path=tmpdir)
+
+        options2 = options.copy()
         game1_objects_names = [info.name for info in game1.infos.values() if info.name is not None]
-        game_file2, game2 = textworld.make(2, 20, 3, 3, {"names_to_exclude": game1_objects_names},
-                                           seed=123, games_dir=tmpdir)
+        options2.grammar.names_to_exclude = game1_objects_names
+        game_file2, game2 = textworld.make(options2, path=tmpdir)
         game2_objects_names = [info.name for info in game2.infos.values() if info.name is not None]
         assert len(set(game1_objects_names) & set(game2_objects_names)) == 0
 
 
 def test_making_game_is_reproducible_with_seed():
-    grammar_flags = {}
     with make_temp_directory(prefix="test_render_wrapper") as tmpdir:
-        game_file1, game1 = textworld.make(2, 20, 3, 3, grammar_flags, seed=123, games_dir=tmpdir)
-        game_file2, game2 = textworld.make(2, 20, 3, 3, grammar_flags, seed=123, games_dir=tmpdir)
+        options = textworld.GameOptions()
+        options.nb_rooms = 2
+        options.nb_objects = 20
+        options.quest_length = 3
+        options.quest_breadth = 3
+        options.seeds = 123
+
+        game_file1, game1 = textworld.make(options, path=tmpdir)
+        options2 = options.copy()
+        game_file2, game2 = textworld.make(options2, path=tmpdir)
         assert game_file1 == game_file2
         assert game1 == game2
         # Make sure they are not the same Python objects.

tests/test_play_generated_games.py

@@ -19,11 +19,15 @@ def test_play_generated_games():
         quest_length = rng.randint(2, 5)
         quest_breadth = rng.randint(3, 7)
         game_seed = rng.randint(0, 65365)
-        grammar_flags = {}  # Default grammar.
 
         with make_temp_directory(prefix="test_play_generated_games") as tmpdir:
-            game_file, game = textworld.make(world_size, nb_objects, quest_length, quest_breadth, grammar_flags,
-                                             seed=game_seed, games_dir=tmpdir)
+            options = textworld.GameOptions()
+            options.nb_rooms = world_size
+            options.nb_objects = nb_objects
+            options.quest_length = quest_length
+            options.quest_breadth = quest_breadth
+            options.seeds = game_seed
+            game_file, game = textworld.make(options, path=tmpdir)
 
             # Solve the game using WalkthroughAgent.
             agent = textworld.agents.WalkthroughAgent()

tests/test_textworld.py

@@ -14,9 +14,12 @@ class TestIntegration(unittest.TestCase):
 
     def setUp(self):
         self.tmpdir = tempfile.mkdtemp(prefix="test_textworld")
-        self.game_file, self.game = textworld.make(world_size=5, nb_objects=10,
-                                                   quest_length=10, grammar_flags={},
-                                                   seed=1234, games_dir=self.tmpdir)
+        options = textworld.GameOptions()
+        options.nb_rooms = 5
+        options.nb_objects = 10
+        options.quest_length = 10
+        options.seeds = 1234
+        self.game_file, self.game = textworld.make(options, path=self.tmpdir)
 
     def tearDown(self):
         shutil.rmtree(self.tmpdir)

tests/test_tw-make.py

@@ -2,6 +2,7 @@
 # Licensed under the MIT license.
 
 import os
+import glob
 from subprocess import check_call
 from os.path import join as pjoin
 
@@ -11,15 +12,51 @@
 
 
 def test_making_a_custom_game():
-    with make_temp_directory(prefix="test_tw-make") as tmpdir:    
+    with make_temp_directory(prefix="test_tw-make") as tmpdir:
         output_folder = pjoin(tmpdir, "gen_games")
-        command = ["tw-make", "custom", "--seed", "1234", "--output", output_folder]
+        game_file = pjoin(output_folder, "game_1234.ulx")
+        command = ["tw-make", "custom", "--seed", "1234", "--output", game_file]
         assert check_call(command) == 0
 
         assert os.path.isdir(output_folder)
-        game_file = pjoin(output_folder, "game_1234.ulx")
         assert os.path.isfile(game_file)
 
         # Solve the game using WalkthroughAgent.
         agent = textworld.agents.WalkthroughAgent()
         textworld.play(game_file, agent=agent, silent=True)
+
+    with make_temp_directory(prefix="test_tw-make") as tmpdir:
+        output_folder = pjoin(tmpdir, "gen_games")
+        game_file = pjoin(output_folder, "game_1234")  # Default extension is .ulx
+        command = ["tw-make", "custom", "--seed", "1234", "--output", game_file]
+        assert check_call(command) == 0
+
+        assert os.path.isdir(output_folder)
+        assert os.path.isfile(game_file + ".ulx")
+
+        # Solve the game using WalkthroughAgent.
+        agent = textworld.agents.WalkthroughAgent()
+        textworld.play(game_file + ".ulx", agent=agent, silent=True)
+
+    with make_temp_directory(prefix="test_tw-make") as tmpdir:
+        output_folder = pjoin(tmpdir, "gen_games", "")
+        command = ["tw-make", "custom", "--seed", "1234", "--output", output_folder]
+        assert check_call(command) == 0
+
+        assert os.path.isdir(output_folder)
+        game_file = glob.glob(pjoin(output_folder, "*.ulx"))[0]
+
+        # Solve the game using WalkthroughAgent.
+        agent = textworld.agents.WalkthroughAgent()
+        textworld.play(game_file, agent=agent, silent=True)
+
+    with make_temp_directory(prefix="test_tw-make") as tmpdir:
+        output_folder = pjoin(tmpdir, "gen_games")
+        command = ["tw-make", "custom", "--seed", "1234", "--output", output_folder]
+        assert check_call(command) == 0
+
+        assert os.path.isfile(output_folder + ".ulx")
+
+        # Solve the game using WalkthroughAgent.
+        agent = textworld.agents.WalkthroughAgent()
+        textworld.play(output_folder + ".ulx", agent=agent, silent=True)

tests/test_tw-play.py

@@ -9,7 +9,13 @@
 
 def test_playing_a_game():
     with make_temp_directory(prefix="test_tw-play") as tmpdir:
-        game_file, _ = textworld.make(5, 10, 5, 4, {}, seed=1234, games_dir=tmpdir)
+        options = textworld.GameOptions()
+        options.nb_rooms = 5
+        options.nb_objects = 10
+        options.quest_length = 5
+        options.quest_breadth = 4
+        options.seeds = 1234
+        game_file, _ = textworld.make(options, path=tmpdir)
 
         command = ["tw-play", "--max-steps", "100", "--mode", "random", game_file]
         assert check_call(command) == 0

textworld/__init__.py

@@ -6,7 +6,7 @@
 from textworld.utils import g_rng
 
 from textworld.core import Environment, GameState, Agent
-from textworld.generator import Game, GameMaker
+from textworld.generator import Game, GameMaker, GameOptions
 
 from textworld.generator import TextworldGenerationWarning