Si usas la librería jQuery de javaScript lo podrías hacer de esta sencilla forma:

<!-- begin snippet: js hide: false console: true babel: false -->

<!-- language: lang-js -->

    $(document).ready(function() {
        $("#add_email").click(function(){
            var contador = $("input[type='email']").length;

            $(this).before('<div><label for="email_'+ contador +'">email:</label><input type="email" id="email_'+ contador +'" /><button type="button" class="delete_email">Del</button></div>');
        });

        $(document).on('click', '.delete_email', function(){
            $(this).parent().remove();
        });
    });

<!-- language: lang-html -->

    <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>

    <form action="page.php" method="post">
        <div>
            <label for="name">Name:</label>
            <input type="text" id="name" />
        </div>

        <div>
            <label for="email">email:</label>
            <input type="email" id="email" />
        </div>
        <div class="button">
            <button type="button" id="add_email">add</button>
        </div>

        <div>
            <label for="msg">Message:</label>
            <textarea id="msg"></textarea>
        </div>

        <div class="button">
            <button type="submit">Send your message</button>
        </div>
    </form>

<!-- end snippet -->